From 4e67fd6128b59b84827511c9b5d4bdd82ee4e236 Mon Sep 17 00:00:00 2001 From: Pranav C Date: Fri, 3 Mar 2023 09:00:53 +0530 Subject: [PATCH] fix: corrections Signed-off-by: Pranav C --- packages/nocodb/src/lib/Noco.ts | 2 +- .../src/lib/controllers/dataApis/helpers.ts | 20 +- .../src/lib/controllers/dataApis/index.ts | 2 +- .../dataController/dataAliasNestedApis.ts | 2 +- .../lib/controllers/projectUserController.ts | 2 +- .../controllers/publicApis/publicDataApis.ts | 2 +- .../controllers/swagger/helpers/getPaths.ts | 0 .../controllers/swagger/helpers/getSchemas.ts | 0 .../swagger/helpers/getSwaggerColumnMetas.ts | 0 .../swagger/helpers/getSwaggerJSON.ts | 0 .../swagger/helpers/templates/headers.ts | 0 .../swagger/helpers/templates/params.ts | 0 .../src/lib/controllers/swagger/redocHtml.ts | 93 - .../lib/controllers/swagger/swaggerApis.ts | 61 - .../lib/controllers/swagger/swaggerHtml.ts | 87 - .../lib/controllers/sync/helpers/EntityMap.ts | 222 -- .../sync/helpers/NocoSyncDestAdapter.ts | 6 - .../sync/helpers/NocoSyncSourceAdapter.ts | 7 - .../lib/controllers/sync/helpers/fetchAT.ts | 238 -- .../src/lib/controllers/sync/helpers/job.ts | 2435 ----------------- .../sync/helpers/readAndProcessData.ts | 338 --- .../lib/controllers/sync/helpers/syncMap.ts | 31 - .../src/lib/controllers/sync/importApis.ts | 138 - .../lib/controllers/sync/syncSourceApis.ts | 69 - .../src/lib/controllers/tableController.ts | 12 +- .../src/lib/controllers/userApi/helpers.ts | 23 - .../src/lib/controllers/userApi/index.ts | 1 - .../controllers/userApi/initAdminFromEnv.ts | 283 -- .../lib/controllers/userApi/initStrategies.ts | 336 --- .../userApi/ui/auth/emailVerify.ts | 70 - .../userApi/ui/auth/resetPassword.ts | 108 - .../ui/emailTemplates/forgotPassword.ts | 171 -- .../userApi/ui/emailTemplates/invite.ts | 208 -- .../userApi/ui/emailTemplates/verify.ts | 207 -- packages/nocodb/src/lib/meta/api/index.ts | 12 +- .../src/lib/services/attachmentService.ts | 3 + .../src/lib/services/projectUserService.ts | 2 +- .../nocodb/src/lib/services/tableService.ts | 109 +- .../src/lib/services/userService/helpers.ts | 2 +- 39 files changed, 144 insertions(+), 5158 deletions(-) delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/redocHtml.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/job.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/importApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/helpers.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/index.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/initStrategies.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts delete mode 100644 packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts diff --git a/packages/nocodb/src/lib/Noco.ts b/packages/nocodb/src/lib/Noco.ts index ca4ae28486..151f211c65 100644 --- a/packages/nocodb/src/lib/Noco.ts +++ b/packages/nocodb/src/lib/Noco.ts @@ -45,7 +45,7 @@ import User from './models/User'; import * as http from 'http'; import weAreHiring from './utils/weAreHiring'; import getInstance from './utils/getInstance'; -import initAdminFromEnv from './controllers/userApi/initAdminFromEnv'; +import initAdminFromEnv from './services/userService/initAdminFromEnv'; const log = debug('nc:app'); require('dotenv').config(); diff --git a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts index 1a7216f5fc..2ca5cd17e1 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/helpers.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/helpers.ts @@ -1,19 +1,19 @@ -import Project from '../../../models/Project'; -import Model from '../../../models/Model'; -import View from '../../../models/View'; -import { NcError } from '../../helpers/catchError'; +import { NcError } from '../../meta/helpers/catchError'; +import Project from '../../models/Project'; +import Model from '../../models/Model'; +import View from '../../models/View'; import { Request } from 'express'; -import Base from '../../../models/Base'; -import NcConnectionMgrv2 from '../../../utils/common/NcConnectionMgrv2'; +import Base from '../../models/Base'; +import NcConnectionMgrv2 from '../../utils/common/NcConnectionMgrv2'; import { isSystemColumn, UITypes } from 'nocodb-sdk'; import * as XLSX from 'xlsx'; -import Column from '../../../models/Column'; -import LookupColumn from '../../../models/LookupColumn'; -import LinkToAnotherRecordColumn from '../../../models/LinkToAnotherRecordColumn'; +import Column from '../../models/Column'; +import LookupColumn from '../../models/LookupColumn'; +import LinkToAnotherRecordColumn from '../../models/LinkToAnotherRecordColumn'; import papaparse from 'papaparse'; -import { dataService } from '../../../services'; +import { dataService } from '../../services'; export async function getViewAndModelFromRequestByAliasOrId( req: | Request<{ projectName: string; tableName: string; viewName?: string }> diff --git a/packages/nocodb/src/lib/controllers/dataApis/index.ts b/packages/nocodb/src/lib/controllers/dataApis/index.ts index 3a4528a5e1..f7b699f340 100644 --- a/packages/nocodb/src/lib/controllers/dataApis/index.ts +++ b/packages/nocodb/src/lib/controllers/dataApis/index.ts @@ -2,7 +2,7 @@ import dataApis from './dataApis'; import oldDataApis from './oldDataApis'; import dataAliasApis from './dataAliasApis'; import bulkDataAliasApis from './bulkDataAliasApis'; -import dataAliasNestedApis from '../../../controllers/dataController/dataAliasNestedApis'; +import dataAliasNestedApis from '../../controllers/dataController/dataAliasNestedApis'; import dataAliasExportApis from './dataAliasExportApis'; export { diff --git a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts index eb11f56d45..d0dc296834 100644 --- a/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts +++ b/packages/nocodb/src/lib/controllers/dataController/dataAliasNestedApis.ts @@ -7,7 +7,7 @@ import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; import { getColumnByIdOrName, getViewAndModelFromRequestByAliasOrId, -} from '../../meta/api/dataApis/helpers'; +} from '../dataApis/helpers'; import { NcError } from '../../meta/helpers/catchError'; import apiMetrics from '../../meta/helpers/apiMetrics'; diff --git a/packages/nocodb/src/lib/controllers/projectUserController.ts b/packages/nocodb/src/lib/controllers/projectUserController.ts index 270db703ed..6be2ff985c 100644 --- a/packages/nocodb/src/lib/controllers/projectUserController.ts +++ b/packages/nocodb/src/lib/controllers/projectUserController.ts @@ -270,7 +270,7 @@ export async function sendInviteEmail( req: any ): Promise { try { - const template = (await import('./userApi/ui/emailTemplates/invite')) + const template = (await import('./userController/ui/emailTemplates/invite')) .default; const emailAdapter = await NcPluginMgrv2.emailAdapter(); diff --git a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts index 17e58cf1a5..792a6b4b61 100644 --- a/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts +++ b/packages/nocodb/src/lib/controllers/publicApis/publicDataApis.ts @@ -15,7 +15,7 @@ import path from 'path'; import { nanoid } from 'nanoid'; import { mimeIcons } from '../../utils/mimeTypes'; import slash from 'slash'; -import { sanitizeUrlPath } from '../attachmentController'; +import { sanitizeUrlPath } from '../../services/attachmentService'; import getAst from '../../db/sql-data-mapper/lib/sql/helpers/getAst'; import { getColumnByIdOrName } from '../dataApis/helpers'; import { NC_ATTACHMENT_FIELD_SIZE } from '../../constants'; diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getPaths.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSchemas.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerColumnMetas.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/getSwaggerJSON.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/headers.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts b/packages/nocodb/src/lib/controllers/swagger/helpers/templates/params.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts b/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts deleted file mode 100644 index cc2682fdac..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/redocHtml.ts +++ /dev/null @@ -1,93 +0,0 @@ -export default ({ - ncSiteUrl, -}: { - ncSiteUrl: string; -}): string => ` - - - NocoDB API Documentation - - - - - - - - -
- - - - -`; diff --git a/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts deleted file mode 100644 index ef44542646..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/swaggerApis.ts +++ /dev/null @@ -1,61 +0,0 @@ -// @ts-ignore -import catchError, { NcError } from '../../meta/helpers/catchError'; -import { Router } from 'express'; -import Model from '../../models/Model'; -import ncMetaAclMw from '../../meta/helpers/ncMetaAclMw'; -import getSwaggerJSON from './helpers/getSwaggerJSON'; -import Project from '../../models/Project'; -import getSwaggerHtml from './swaggerHtml'; -import getRedocHtml from './redocHtml'; - -async function swaggerJson(req, res) { - const project = await Project.get(req.params.projectId); - - if (!project) NcError.notFound(); - - const models = await Model.list({ - project_id: req.params.projectId, - base_id: null, - }); - - const swagger = await getSwaggerJSON(project, models); - - swagger.servers = [ - { - url: req.ncSiteUrl, - }, - { - url: '{customUrl}', - variables: { - customUrl: { - default: req.ncSiteUrl, - description: 'Provide custom nocodb app base url', - }, - }, - }, - ] as any; - - res.json(swagger); -} - -function swaggerHtml(_, res) { - res.send(getSwaggerHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); -} - -function redocHtml(_, res) { - res.send(getRedocHtml({ ncSiteUrl: process.env.NC_PUBLIC_URL || '' })); -} - -const router = Router({ mergeParams: true }); - -// todo: auth -router.get( - '/api/v1/db/meta/projects/:projectId/swagger.json', - ncMetaAclMw(swaggerJson, 'swaggerJson') -); - -router.get('/api/v1/db/meta/projects/:projectId/swagger', swaggerHtml); - -router.get('/api/v1/db/meta/projects/:projectId/redoc', redocHtml); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts b/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts deleted file mode 100644 index 27f6bef73d..0000000000 --- a/packages/nocodb/src/lib/controllers/swagger/swaggerHtml.ts +++ /dev/null @@ -1,87 +0,0 @@ -export default ({ - ncSiteUrl, -}: { - ncSiteUrl: string; -}): string => ` - - - NocoDB : API Docs - - - - - - -
- - - -`; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts deleted file mode 100644 index bd0bdd86ca..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/EntityMap.ts +++ /dev/null @@ -1,222 +0,0 @@ -import sqlite3 from 'sqlite3'; -import { Readable } from 'stream'; - -class EntityMap { - initialized: boolean; - cols: string[]; - db: any; - - constructor(...args) { - this.initialized = false; - this.cols = args.map((arg) => processKey(arg)); - this.db = new Promise((resolve, reject) => { - const db = new sqlite3.Database(':memory:'); - - const colStatement = - this.cols.length > 0 - ? this.cols.join(' TEXT, ') + ' TEXT' - : 'mappingPlaceholder TEXT'; - db.run(`CREATE TABLE mapping (${colStatement})`, (err) => { - if (err) { - console.log(err); - reject(err); - } - resolve(db); - }); - }); - } - - async init() { - if (!this.initialized) { - this.db = await this.db; - this.initialized = true; - } - } - - destroy() { - if (this.initialized && this.db) { - this.db.close(); - } - } - - async addRow(row) { - if (!this.initialized) { - throw 'Please initialize first!'; - } - - const cols = Object.keys(row).map((key) => processKey(key)); - const colStatement = cols.map((key) => `'${key}'`).join(', '); - const questionMarks = cols.map(() => '?').join(', '); - - const promises = []; - - for (const col of cols.filter((col) => !this.cols.includes(col))) { - promises.push( - new Promise((resolve, reject) => { - this.db.run(`ALTER TABLE mapping ADD '${col}' TEXT;`, (err) => { - if (err) { - console.log(err); - reject(err); - } - this.cols.push(col); - resolve(true); - }); - }) - ); - } - - await Promise.all(promises); - - const values = Object.values(row).map((val) => { - if (typeof val === 'object') { - return `JSON::${JSON.stringify(val)}`; - } - return val; - }); - - return new Promise((resolve, reject) => { - this.db.run( - `INSERT INTO mapping (${colStatement}) VALUES (${questionMarks})`, - values, - (err) => { - if (err) { - console.log(err); - reject(err); - } - resolve(true); - } - ); - }); - } - - getRow(col, val, res = []): Promise> { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - col = processKey(col); - res = res.map((r) => processKey(r)); - this.db.get( - `SELECT ${ - res.length ? res.join(', ') : '*' - } FROM mapping WHERE ${col} = ?`, - [val], - (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - if (rs) { - rs = processResponseRow(rs); - } - resolve(rs); - } - ); - }); - } - - getCount(): Promise { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - this.db.get(`SELECT COUNT(*) as count FROM mapping`, (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - resolve(rs.count); - }); - }); - } - - getStream(res = []): DBStream { - if (!this.initialized) { - throw 'Please initialize first!'; - } - res = res.map((r) => processKey(r)); - return new DBStream( - this.db, - `SELECT ${res.length ? res.join(', ') : '*'} FROM mapping` - ); - } - - getLimit(limit, offset, res = []): Promise[]> { - if (!this.initialized) { - throw 'Please initialize first!'; - } - return new Promise((resolve, reject) => { - res = res.map((r) => processKey(r)); - this.db.all( - `SELECT ${ - res.length ? res.join(', ') : '*' - } FROM mapping LIMIT ${limit} OFFSET ${offset}`, - (err, rs) => { - if (err) { - console.log(err); - reject(err); - } - for (let row of rs) { - row = processResponseRow(row); - } - resolve(rs); - } - ); - }); - } -} - -class DBStream extends Readable { - db: any; - stmt: any; - sql: any; - - constructor(db, sql) { - super({ objectMode: true }); - this.db = db; - this.sql = sql; - this.stmt = this.db.prepare(this.sql); - this.on('end', () => this.stmt.finalize()); - } - - _read() { - let stream = this; - this.stmt.get(function (err, result) { - if (err) { - stream.emit('error', err); - } else { - if (result) { - result = processResponseRow(result); - } - stream.push(result || null); - } - }); - } -} - -function processResponseRow(res: any) { - for (const key of Object.keys(res)) { - if (res[key] && res[key].startsWith('JSON::')) { - try { - res[key] = JSON.parse(res[key].replace('JSON::', '')); - } catch (e) { - console.log(e); - } - } - if (revertKey(key) !== key) { - res[revertKey(key)] = res[key]; - delete res[key]; - } - } - return res; -} - -function processKey(key) { - return key.replace(/'/g, "''").replace(/[A-Z]/g, (match) => `_${match}`); -} - -function revertKey(key) { - return key.replace(/''/g, "'").replace(/_[A-Z]/g, (match) => match[1]); -} - -export default EntityMap; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts deleted file mode 100644 index 8af4a493c5..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncDestAdapter.ts +++ /dev/null @@ -1,6 +0,0 @@ -export abstract class NocoSyncSourceAdapter { - public abstract init(): Promise; - public abstract destProjectWrite(): Promise; - public abstract destSchemaWrite(): Promise; - public abstract destDataWrite(): Promise; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts b/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts deleted file mode 100644 index a419d1c24d..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/NocoSyncSourceAdapter.ts +++ /dev/null @@ -1,7 +0,0 @@ -export abstract class NocoSyncSourceAdapter { - public abstract init(): Promise; - public abstract srcSchemaGet(): Promise; - public abstract srcDataLoad(): Promise; - public abstract srcDataListen(): Promise; - public abstract srcDataPoll(): Promise; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts b/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts deleted file mode 100644 index 4ada40bd17..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/fetchAT.ts +++ /dev/null @@ -1,238 +0,0 @@ -const axios = require('axios').default; - -const info: any = { - initialized: false, -}; - -async function initialize(shareId) { - info.cookie = ''; - const url = `https://airtable.com/${shareId}`; - - try { - const hreq = await axios - .get(url, { - headers: { - accept: - 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'document', - 'sec-fetch-mode': 'navigate', - 'sec-fetch-site': 'none', - 'sec-fetch-user': '?1', - 'upgrade-insecure-requests': '1', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - }, - referrerPolicy: 'strict-origin-when-cross-origin', - body: null, - method: 'GET', - }) - .then((response) => { - for (const ck of response.headers['set-cookie']) { - info.cookie += ck.split(';')[0] + '; '; - } - return response.data; - }) - .catch(() => { - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - - info.headers = JSON.parse( - hreq.match(/(?<=var headers =)(.*)(?=;)/g)[0].trim() - ); - info.link = unicodeToChar(hreq.match(/(?<=fetch\(")(.*)(?=")/g)[0].trim()); - info.baseInfo = decodeURIComponent(info.link) - .match(/{(.*)}/g)[0] - .split('&') - .reduce((result, el) => { - try { - return Object.assign( - result, - JSON.parse(el.includes('=') ? el.split('=')[1] : el) - ); - } catch (e) { - if (el.includes('=')) { - return Object.assign(result, { - [el.split('=')[0]]: el.split('=')[1], - }); - } - } - }, {}); - info.baseId = info.baseInfo.applicationId; - info.initialized = true; - } catch (e) { - console.log(e); - info.initialized = false; - if (e.message) { - throw e; - } else { - throw { - message: - 'Error processing Shared Base :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - } - } -} - -async function read() { - if (info.initialized) { - const resreq = await axios('https://airtable.com' + info.link, { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrerPolicy: 'no-referrer', - body: null, - method: 'GET', - }) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Reading :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - - return { - schema: resreq.data, - baseId: info.baseId, - baseInfo: info.baseInfo, - }; - } else { - throw { - message: 'Error Initializing :: please try again !!', - }; - } -} - -async function readView(viewId) { - if (info.initialized) { - const resreq = await axios( - `https://airtable.com/v0.3/view/${viewId}/readData?` + - `stringifiedObjectParams=${encodeURIComponent('{}')}&requestId=${ - info.baseInfo.requestId - }&accessPolicy=${encodeURIComponent( - JSON.stringify({ - allowedActions: info.baseInfo.allowedActions, - shareId: info.baseInfo.shareId, - applicationId: info.baseInfo.applicationId, - generationNumber: info.baseInfo.generationNumber, - expires: info.baseInfo.expires, - signature: info.baseInfo.signature, - }) - )}`, - { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrerPolicy: 'no-referrer', - body: null, - method: 'GET', - } - ) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Reading View :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - }); - return { view: resreq.data }; - } else { - throw { - message: 'Error Initializing :: please try again !!', - }; - } -} - -async function readTemplate(templateId) { - if (!info.initialized) { - await initialize('shrO8aYf3ybwSdDKn'); - } - const resreq = await axios( - `https://www.airtable.com/v0.3/exploreApplications/${templateId}`, - { - headers: { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', - 'sec-ch-ua': - '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Linux"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'User-Agent': - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36', - 'x-time-zone': 'Europe/Berlin', - cookie: info.cookie, - ...info.headers, - }, - referrer: 'https://www.airtable.com/', - referrerPolicy: 'same-origin', - body: null, - method: 'GET', - mode: 'cors', - credentials: 'include', - } - ) - .then((response) => { - return response.data; - }) - .catch(() => { - throw { - message: - 'Error Fetching :: Ensure www.airtable.com/templates/featured/ is accessible.', - }; - }); - return { template: resreq }; -} - -function unicodeToChar(text) { - return text.replace(/\\u[\dA-F]{4}/gi, function (match) { - return String.fromCharCode(parseInt(match.replace(/\\u/g, ''), 16)); - }); -} - -export default { - initialize, - read, - readView, - readTemplate, -}; diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts b/packages/nocodb/src/lib/controllers/sync/helpers/job.ts deleted file mode 100644 index 9e55beb4bd..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/job.ts +++ /dev/null @@ -1,2435 +0,0 @@ -import { T } from 'nc-help'; -import FetchAT from './fetchAT'; -import { UITypes } from 'nocodb-sdk'; -// import * as sMap from './syncMap'; - -import { Api } from 'nocodb-sdk'; - -import Airtable from 'airtable'; -import jsonfile from 'jsonfile'; -import hash from 'object-hash'; -import { promisify } from 'util'; - -import dayjs from 'dayjs'; -import tinycolor from 'tinycolor2'; -import { importData, importLTARData } from './readAndProcessData'; - -import EntityMap from './EntityMap'; - -const writeJsonFileAsync = promisify(jsonfile.writeFile); - -const selectColors = { - // normal - blue: '#cfdfff', - cyan: '#d0f0fd', - teal: '#c2f5e9', - green: '#d1f7c4', - orange: '#fee2d5', - yellow: '#ffeab6', - red: '#ffdce5', - pink: '#ffdaf6', - purple: '#ede2fe', - gray: '#eee', - // medium - blueMedium: '#9cc7ff', - cyanMedium: '#77d1f3', - tealMedium: '#72ddc3', - greenMedium: '#93e088', - orangeMedium: '#ffa981', - yellowMedium: '#ffd66e', - redMedium: '#ff9eb7', - pinkMedium: '#f99de2', - purpleMedium: '#cdb0ff', - grayMedium: '#ccc', - // dark - blueDark: '#2d7ff9', - cyanDark: '#18bfff', - tealDark: '#20d9d2', - greenDark: '#20c933', - orangeDark: '#ff6f2c', - yellowDark: '#fcb400', - redDark: '#f82b60', - pinkDark: '#ff08c2', - purpleDark: '#8b46ff', - grayDark: '#666', - // darker - blueDarker: '#2750ae', - cyanDarker: '#0b76b7', - tealDarker: '#06a09b', - greenDarker: '#338a17', - orangeDarker: '#d74d26', - yellowDarker: '#b87503', - redDarker: '#ba1e45', - pinkDarker: '#b2158b', - purpleDarker: '#6b1cb0', - grayDarker: '#444', -}; - -export default async ( - syncDB: AirtableSyncConfig, - progress: (data: { msg?: string; level?: any }) => void -) => { - const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent'); - await sMapEM.init(); - - const sMap = { - // static mapping records between aTblId && ncId - async addToMappingTbl(aTblId, ncId, ncName, ncParent?) { - await sMapEM.addRow({ aTblId, ncId, ncName, ncParent }); - }, - - // get NcID from airtable ID - async getNcIdFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncId']))?.ncId; - }, - - // get nc Parent from airtable ID - async getNcParentFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncParent']))?.ncParent; - }, - - // get nc-title from airtable ID - async getNcNameFromAtId(aId) { - return (await sMapEM.getRow('aTblId', aId, ['ncName']))?.ncName; - }, - }; - - function logBasic(log) { - progress({ level: 0, msg: log }); - } - - function logDetailed(log) { - if (debugMode) progress({ level: 1, msg: log }); - } - - const perfStats = []; - function recordPerfStart() { - if (!debugMode) return 0; - return Date.now(); - } - function recordPerfStats(start, event) { - if (!debugMode) return; - const duration = Date.now() - start; - perfStats.push({ d: duration, e: event }); - } - - let base, baseId; - const start = Date.now(); - const enableErrorLogs = false; - const generate_migrationStats = true; - const debugMode = false; - let api: Api; - let g_aTblSchema = []; - let ncCreatedProjectSchema: any = {}; - const ncLinkMappingTable: any[] = []; - const nestedLookupTbl: any[] = []; - const nestedRollupTbl: any[] = []; - const ncSysFields = { id: 'ncRecordId', hash: 'ncRecordHash' }; - const storeLinks = false; - const ncLinkDataStore: any = {}; - const insertedAssocRef: any = {}; - - const atNcAliasRef: { - [ncTableId: string]: { - [ncTitle: string]: string; - }; - } = {}; - - const uniqueTableNameGen = getUniqueNameGenerator('sheet'); - - // run time counter (statistics) - const rtc = { - sort: 0, - filter: 0, - view: { - total: 0, - grid: 0, - gallery: 0, - form: 0, - }, - fetchAt: { - count: 0, - time: 0, - }, - migrationSkipLog: { - count: 0, - log: [], - }, - data: { - records: 0, - nestedLinks: 0, - }, - }; - - function updateMigrationSkipLog(tbl, col, type, reason?) { - rtc.migrationSkipLog.count++; - rtc.migrationSkipLog.log.push( - `tn[${tbl}] cn[${col}] type[${type}] :: ${reason}` - ); - } - - // mapping table - // - - async function getAirtableSchema(sDB) { - const start = Date.now(); - - if (!sDB.shareId) - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - - if (sDB.shareId.startsWith('exp')) { - const template = await FetchAT.readTemplate(sDB.shareId); - await FetchAT.initialize(template.template.exploreApplication.shareId); - } else { - await FetchAT.initialize(sDB.shareId); - } - const ft = await FetchAT.read(); - const duration = Date.now() - start; - rtc.fetchAt.count++; - rtc.fetchAt.time += duration; - - if (!ft.baseId) { - throw { - message: - 'Invalid Shared Base ID :: Ensure www.airtable.com/ is accessible. Refer https://bit.ly/3x0OdXI for details', - }; - } - - const file = ft.schema; - baseId = ft.baseId; - base = new Airtable({ apiKey: sDB.apiKey }).base(baseId); - // store copy of airtable schema globally - g_aTblSchema = file.tableSchemas; - - if (debugMode) - await writeJsonFileAsync('aTblSchema.json', ft, { spaces: 2 }); - - return file; - } - - async function getViewData(viewId) { - const start = Date.now(); - const ft = await FetchAT.readView(viewId); - const duration = Date.now() - start; - rtc.fetchAt.count++; - rtc.fetchAt.time += duration; - - if (debugMode) - await writeJsonFileAsync(`${viewId}.json`, ft, { spaces: 2 }); - return ft.view; - } - - function getRootDbType() { - return ncCreatedProjectSchema?.bases.find((el) => el.id === syncDB.baseId) - ?.type; - } - - // base mapping table - const aTblNcTypeMap = { - foreignKey: UITypes.LinkToAnotherRecord, - text: UITypes.SingleLineText, - multilineText: UITypes.LongText, - richText: UITypes.LongText, - multipleAttachment: UITypes.Attachment, - checkbox: UITypes.Checkbox, - multiSelect: UITypes.MultiSelect, - select: UITypes.SingleSelect, - collaborator: UITypes.Collaborator, - multiCollaborator: UITypes.Collaborator, - date: UITypes.Date, - phone: UITypes.PhoneNumber, - number: UITypes.Decimal, - rating: UITypes.Rating, - formula: UITypes.Formula, - rollup: UITypes.Rollup, - count: UITypes.Count, - lookup: UITypes.Lookup, - autoNumber: UITypes.AutoNumber, - barcode: UITypes.SingleLineText, - button: UITypes.Button, - }; - - //----------------------------------------------------------------------------- - // aTbl helper routines - // - - function nc_sanitizeName(name) { - // replace all special characters by _ - return name.replace(/\W+/g, '_').trim(); - } - - function nc_getSanitizedColumnName(table, name) { - let col_name = nc_sanitizeName(name); - - // truncate to 60 chars if character if exceeds above 60 - col_name = col_name?.slice(0, 60); - - // for knex, replace . with _ - const col_alias = name.trim().replace(/\./g, '_'); - - // check if already a column exists with same name? - const duplicateTitle = table.columns.find( - (x) => x.title?.toLowerCase() === col_alias?.toLowerCase() - ); - const duplicateColumn = table.columns.find( - (x) => x.column_name?.toLowerCase() === col_name?.toLowerCase() - ); - if (duplicateTitle) { - if (enableErrorLogs) console.log(`## Duplicate title ${col_alias}`); - } - - return { - // kludge: error observed in Nc with space around column-name - title: col_alias + (duplicateTitle ? '_2' : ''), - column_name: col_name + (duplicateColumn ? '_2' : ''), - }; - } - - // aTbl: retrieve table name from table ID - // - // @ts-ignore - function aTbl_getTableName(tblId) { - const sheetObj = g_aTblSchema.find((tbl) => tbl.id === tblId); - return { - tn: sheetObj.name, - }; - } - - const ncSchema = { - tables: [], - tablesById: {}, - }; - - // aTbl: retrieve column name from column ID - // - function aTbl_getColumnName(colId): any { - for (let i = 0; i < g_aTblSchema.length; i++) { - const sheetObj = g_aTblSchema[i]; - const column = sheetObj.columns.find((col) => col.id === colId); - if (column !== undefined) - return { - tn: sheetObj.name, - cn: column.name, - }; - } - } - - // nc dump schema - // - // @ts-ignore - async function nc_DumpTableSchema() { - console.log('['); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); - for (let i = 0; i < ncTblList.list.length; i++) { - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - console.log(JSON.stringify(ncTbl, null, 2)); - console.log(','); - } - console.log(']'); - } - - // retrieve nc column schema from using aTbl field ID as reference - // - async function nc_getColumnSchema(aTblFieldId) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let aTblField = aTbl_getColumnName(aTblFieldId); - // let ncTblId = ncTblList.list.filter(x => x.title === aTblField.tn)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // let ncCol = ncTbl.columns.find(x => x.title === aTblField.cn); - // return ncCol; - - const ncTblId = await sMap.getNcParentFromAtId(aTblFieldId); - const ncColId = await sMap.getNcIdFromAtId(aTblFieldId); - - // not migrated column, skip - if (ncColId === undefined || ncTblId === undefined) return 0; - - return ncSchema.tablesById[ncTblId].columns.find((x) => x.id === ncColId); - } - - // retrieve nc table schema using table name - // optimize: create a look-up table & re-use information - // - async function nc_getTableSchema(tableName) { - // let ncTblList = await api.dbTable.list(ncCreatedProjectSchema.id); - // let ncTblId = ncTblList.list.filter(x => x.title === tableName)[0].id; - // let ncTbl = await api.dbTable.read(ncTblId); - // return ncTbl; - - return ncSchema.tables.find((x) => x.title === tableName); - } - - // delete project if already exists - async function init({ - projectName, - }: { - projectName?: string; - projectId?: string; - }) { - // delete 'sample' project if already exists - const x = await api.project.list(); - - const sampleProj = x.list.find((a) => a.title === projectName); - if (sampleProj) { - await api.project.delete(sampleProj.id); - } - logDetailed('Init'); - } - - // map UIDT - // - function getNocoType(col) { - // start with default map - let ncType = aTblNcTypeMap[col.type]; - - // types email & url are marked as text - // types currency & percent, duration are marked as number - // types createTime & modifiedTime are marked as formula - - switch (col.type) { - case 'text': - if (col.typeOptions?.validatorName === 'email') ncType = UITypes.Email; - else if (col.typeOptions?.validatorName === 'url') ncType = UITypes.URL; - break; - - case 'number': - // kludge: currency validation error with decimal places - if (col.typeOptions?.format === 'percentV2') ncType = UITypes.Percent; - else if (col.typeOptions?.format === 'duration') - ncType = UITypes.Duration; - else if (col.typeOptions?.format === 'currency') - ncType = UITypes.Currency; - else if (col.typeOptions?.precision > 0) ncType = UITypes.Decimal; - break; - - case 'formula': - if (col.typeOptions?.formulaTextParsed === 'CREATED_TIME()') - ncType = UITypes.DateTime; - else if (col.typeOptions?.formulaTextParsed === 'LAST_MODIFIED_TIME()') - ncType = UITypes.DateTime; - break; - - case 'computation': - if (col.typeOptions?.resultType === 'collaborator') - ncType = UITypes.Collaborator; - break; - - case 'date': - if (col.typeOptions?.isDateTime) ncType = UITypes.DateTime; - break; - - // case 'barcode': - // case 'button': - // ncType = UITypes.SingleLineText; - // break; - } - - return ncType; - } - - // retrieve additional options associated with selected data types - // - async function getNocoTypeOptions(col: any): Promise { - switch (col.type) { - case 'select': - case 'multiSelect': { - // prepare options list in CSV format - // note: NC doesn't allow comma's in options - // - const options = []; - let order = 1; - for (const [, value] of Object.entries(col.typeOptions.choices)) { - // replace commas with dot for multiselect - if (col.type === 'multiSelect') { - (value as any).name = (value as any).name.replace(/,/g, '.'); - } - // we don't allow empty records, placeholder instead - if ((value as any).name === '') { - (value as any).name = 'nc_empty'; - } - // enumerate duplicates (we don't allow them) - // TODO fix record mapping (this causes every record to map first option, we can't handle them using data api as they don't provide option id within data we might instead get the correct mapping from schema file ) - let dupNo = 1; - const defaultName = (value as any).name; - while ( - options.find( - (el) => - el.title.toLowerCase() === (value as any).name.toLowerCase() - ) - ) { - (value as any).name = `${defaultName}_${dupNo++}`; - } - options.push({ - order: order++, - title: (value as any).name, - color: selectColors[(value as any).color] - ? selectColors[(value as any).color] - : tinycolor.random().toHexString(), - }); - - await sMap.addToMappingTbl( - (value as any).id, - undefined, - (value as any).name - ); - } - return { type: col.type, data: options }; - } - default: - return { type: undefined }; - } - } - - // convert to Nc schema (basic, excluding relations) - // - async function tablesPrepare(tblSchema: any[]) { - const tables: any[] = []; - - for (let i = 0; i < tblSchema.length; ++i) { - const table: any = {}; - - if (syncDB.options.syncViews) { - rtc.view.total += tblSchema[i].views.reduce( - (acc, cur) => - ['grid', 'form', 'gallery'].includes(cur.type) ? ++acc : acc, - 0 - ); - } else { - rtc.view.total = tblSchema.length; - } - - // Enable to use aTbl identifiers as is: table.id = tblSchema[i].id; - table.title = tblSchema[i].name; - let sanitizedName = nc_sanitizeName(tblSchema[i].name); - - // truncate to 50 chars if character if exceeds above 50 - // upto 64 should be fine but we are keeping it to 50 since - // meta project adds prefix as well - sanitizedName = sanitizedName?.slice(0, 50); - - // check for duplicate and populate a unique name if already exist - table.table_name = uniqueTableNameGen(sanitizedName); - - const uniqueColNameGen = getUniqueNameGenerator('field'); - table.columns = []; - const sysColumns = [ - { - title: ncSysFields.id, - column_name: ncSysFields.id, - uidt: UITypes.ID, - meta: { - ag: 'nc', - }, - }, - { - title: ncSysFields.hash, - column_name: ncSysFields.hash, - uidt: UITypes.SingleLineText, - system: true, - }, - ]; - - for (let j = 0; j < tblSchema[i].columns.length; j++) { - const col = tblSchema[i].columns[j]; - - // skip link, lookup, rollup fields in this iteration - if (['foreignKey', 'lookup', 'rollup'].includes(col.type)) { - continue; - } - - // base column schema - const ncName: any = nc_getSanitizedColumnName(table, col.name); - const ncCol: any = { - // Enable to use aTbl identifiers as is: id: col.id, - title: ncName.title, - column_name: uniqueColNameGen(ncName.column_name), - uidt: getNocoType(col), - }; - - // not supported datatype: pure formula field - // allow formula based computed fields (created time/ modified time to go through) - if (ncCol.uidt === UITypes.Formula) { - updateMigrationSkipLog( - tblSchema[i].name, - ncName.title, - col.type, - 'column type not supported' - ); - continue; - } - - // populate cdf (column default value) if configured - // if (col?.default) { - // if (typeof col.default === 'string') - // ncCol.cdf = `'${col.default.replace?.(/'/g, "\\'")}'`; - // else ncCol.cdf = col.default; - // } - - // change from default 'tinytext' as airtable allows more than 255 characters - // for single line text column type - if (col.type === 'text') ncCol.dt = 'text'; - - // #fix-2363-decimal-out-of-range - if (['sqlite3', 'mysql2'].includes(getRootDbType())) { - if (ncCol.uidt === UITypes.Decimal) { - ncCol.dt = 'double'; - ncCol.dtxp = 22; - ncCol.dtxs = '2'; - } - } - - // additional column parameters when applicable - const colOptions = await getNocoTypeOptions(col); - - switch (colOptions.type) { - case 'select': - case 'multiSelect': - ncCol.colOptions = { - options: [...colOptions.data], - }; - - if (['mysql', 'mysql2'].includes(getRootDbType())) { - // if options are empty, configure '' as an option - ncCol.dtxp = - colOptions.data - .map((el) => `'${el.title.replace(/'/gi, "''")}'`) - .join(',') || "''"; - } - - break; - case undefined: - break; - } - table.columns.push(ncCol); - } - table.columns.push(sysColumns[0]); - table.columns.push(sysColumns[1]); - - tables.push(table); - } - return tables; - } - - async function nocoCreateBaseSchema(aTblSchema) { - // base schema preparation: exclude - const tables: any[] = await tablesPrepare(aTblSchema); - - // for each table schema, create nc table - for (let idx = 0; idx < tables.length; idx++) { - logBasic(`:: [${idx + 1}/${tables.length}] ${tables[idx].title}`); - - logDetailed(`NC API: base.tableCreate ${tables[idx].title}`); - - let _perfStart = recordPerfStart(); - const table: any = await api.base.tableCreate( - ncCreatedProjectSchema.id, - syncDB.baseId, - tables[idx] - ); - recordPerfStats(_perfStart, 'dbTable.create'); - - updateNcTblSchema(table); - - // update mapping table - await sMap.addToMappingTbl(aTblSchema[idx].id, table.id, table.title); - for (let colIdx = 0; colIdx < table.columns.length; colIdx++) { - const aId = aTblSchema[idx].columns.find( - (x) => - x.name.trim().replace(/\./g, '_') === table.columns[colIdx].title - )?.id; - if (aId) - await sMap.addToMappingTbl( - aId, - table.columns[colIdx].id, - table.columns[colIdx].title, - table.id - ); - } - - // update default view name- to match it to airtable view name - logDetailed(`NC API: dbView.list ${table.id}`); - _perfStart = recordPerfStart(); - const view = await api.dbView.list(table.id); - recordPerfStats(_perfStart, 'dbView.list'); - - const aTbl_grid = aTblSchema[idx].views.find((x) => x.type === 'grid'); - logDetailed(`NC API: dbView.update ${view.list[0].id} ${aTbl_grid.name}`); - _perfStart = recordPerfStart(); - await api.dbView.update(view.list[0].id, { - title: aTbl_grid.name, - }); - recordPerfStats(_perfStart, 'dbView.update'); - - await updateNcTblSchemaById(table.id); - - await sMap.addToMappingTbl( - aTbl_grid.id, - table.views[0].id, - aTbl_grid.name, - table.id - ); - } - - return tables; - } - - async function nocoCreateLinkToAnotherRecord(aTblSchema) { - // Link to another RECORD - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblLinkColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'foreignKey' - ); - - // Link columns exist - // - if (aTblLinkColumns.length) { - for (let i = 0; i < aTblLinkColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Links :: [${i + 1}/${ - aTblLinkColumns.length - }] ${aTblSchema[idx].name}` - ); - - // for self links, there is no symmetric column - { - const src = aTbl_getColumnName(aTblLinkColumns[i].id); - const dst = aTbl_getColumnName( - aTblLinkColumns[i].typeOptions?.symmetricColumnId - ); - logDetailed( - `LTAR ${src.tn}:${src.cn} <${aTblLinkColumns[i].typeOptions.relationship}> ${dst?.tn}:${dst?.cn}` - ); - } - - // check if link already established? - if (!nc_isLinkExists(aTblLinkColumns[i].id)) { - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - - // find child table name from symmetric column ID specified - // self link, symmetricColumnId field will be undefined - const childTable = aTbl_getColumnName( - aTblLinkColumns[i].typeOptions?.symmetricColumnId - ); - - // retrieve child table ID (nc) from table name - let childTableId = srcTableId; - if (childTable) { - childTableId = (await nc_getTableSchema(childTable.tn)).id; - } - - // check if already a column exists with this name? - let _perfStart = recordPerfStart(); - const srcTbl: any = await api.dbTable.read(srcTableId); - recordPerfStats(_perfStart, 'dbTable.read'); - - // create link - const ncName = nc_getSanitizedColumnName( - srcTbl, - aTblLinkColumns[i].name - ); - - // LTAR alias ref to AT - atNcAliasRef[srcTbl.id] = atNcAliasRef[srcTbl.id] || {}; - atNcAliasRef[srcTbl.id][ncName.title] = aTblLinkColumns[i].name; - - logDetailed( - `NC API: dbTableColumn.create LinkToAnotherRecord ${ncName.title}` - ); - _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.LinkToAnotherRecord, - title: ncName.title, - column_name: ncName.column_name, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - // aTblLinkColumns[i].typeOptions.relationship === 'many' - // ? 'mm' - // : 'hm' - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === ncName.title - )?.id; - await sMap.addToMappingTbl( - aTblLinkColumns[i].id, - ncId, - ncName.title, - ncTbl.id - ); - - // store link information in separate table - // this information will be helpful in identifying relation pair - const link = { - nc: { - title: ncName.title, - parentId: srcTableId, - childId: childTableId, - type: 'mm', - }, - aTbl: { - tblId: aTblSchema[idx].id, - ...aTblLinkColumns[i], - }, - }; - - ncLinkMappingTable.push(link); - } else { - // if link already exists, we need to change name of linked column - // to what is represented in airtable - - // 1. extract associated link information from link table - // 2. retrieve parent table information (source) - // 3. using foreign parent & child column ID, find associated mapping in child table - // 4. update column name - const x = ncLinkMappingTable.findIndex( - (x) => - x.aTbl.tblId === - aTblLinkColumns[i].typeOptions.foreignTableId && - x.aTbl.id === aTblLinkColumns[i].typeOptions.symmetricColumnId - ); - - let _perfStart = recordPerfStart(); - const childTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.childId - ); - recordPerfStats(_perfStart, 'dbTable.read'); - - _perfStart = recordPerfStart(); - const parentTblSchema: any = await api.dbTable.read( - ncLinkMappingTable[x].nc.parentId - ); - recordPerfStats(_perfStart, 'dbTable.read'); - - // fix me - // let childTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.childId] - // let parentTblSchema = ncSchema.tablesById[ncLinkMappingTable[x].nc.parentId] - - let parentLinkColumn = parentTblSchema.columns.find( - (col) => col.title === ncLinkMappingTable[x].nc.title - ); - - if (parentLinkColumn === undefined) { - updateMigrationSkipLog( - parentTblSchema?.title, - ncLinkMappingTable[x].nc.title, - UITypes.LinkToAnotherRecord, - 'Link error' - ); - continue; - } - - // hack // fix me - if (parentLinkColumn.uidt !== 'LinkToAnotherRecord') { - parentLinkColumn = parentTblSchema.columns.find( - (col) => col.title === ncLinkMappingTable[x].nc.title + '_2' - ); - } - - let childLinkColumn: any = {}; - - if (parentLinkColumn.colOptions.type == 'hm') { - // for hm: - // mapping between child & parent column id is direct - // - childLinkColumn = childTblSchema.columns.find( - (col) => - col.uidt === UITypes.LinkToAnotherRecord && - col.colOptions.fk_child_column_id === - parentLinkColumn.colOptions.fk_child_column_id && - col.colOptions.fk_parent_column_id === - parentLinkColumn.colOptions.fk_parent_column_id - ); - } else { - // for mm: - // mapping between child & parent column id is inverted - // - childLinkColumn = childTblSchema.columns.find( - (col) => - col.uidt === UITypes.LinkToAnotherRecord && - col.colOptions.fk_child_column_id === - parentLinkColumn.colOptions.fk_parent_column_id && - col.colOptions.fk_parent_column_id === - parentLinkColumn.colOptions.fk_child_column_id && - col.colOptions.fk_mm_model_id === - parentLinkColumn.colOptions.fk_mm_model_id - ); - } - - // check if already a column exists with this name? - const duplicate = childTblSchema.columns.find( - (x) => x.title === aTblLinkColumns[i].name - ); - const suffix = duplicate ? '_2' : ''; - if (duplicate) - if (enableErrorLogs) - console.log(`## Duplicate ${aTblLinkColumns[i].name}`); - - // rename - // note that: current rename API requires us to send all parameters, - // not just title being renamed - const ncName = nc_getSanitizedColumnName( - childTblSchema, - aTblLinkColumns[i].name - ); - - logDetailed( - `NC API: dbTableColumn.update rename symmetric column ${ncName.title}` - ); - _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.update( - childLinkColumn.id, - { - ...childLinkColumn, - title: ncName.title, - column_name: ncName.column_name, - } - ); - recordPerfStats(_perfStart, 'dbTableColumn.update'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblLinkColumns[i].name + suffix - )?.id; - await sMap.addToMappingTbl( - aTblLinkColumns[i].id, - ncId, - aTblLinkColumns[i].name + suffix, - ncTbl.id - ); - } - } - } - } - } - - async function nocoCreateLookups(aTblSchema) { - // LookUps - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'lookup' - ); - - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - if (aTblColumns.length) { - // Lookup - for (let i = 0; i < aTblColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Lookup :: [${ - i + 1 - }/${aTblColumns.length}] ${aTblSchema[idx].name}` - ); - - // something is not right, skip - if ( - aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - if (enableErrorLogs) - console.log(`## Invalid column IDs mapped; skip`); - - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'invalid column ID in dependency list' - ); - continue; - } - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - - if ( - ncLookupColumnId === undefined || - ncRelationColumnId === undefined - ) { - aTblColumns[i]['srcTableId'] = srcTableId; - nestedLookupTbl.push(aTblColumns[i]); - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - aTblColumns[i].name - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblColumns[i].name - )?.id; - await sMap.addToMappingTbl( - aTblColumns[i].id, - ncId, - aTblColumns[i].name, - ncTbl.id - ); - } - } - } - - let level = 2; - let nestedCnt = 0; - while (nestedLookupTbl.length) { - // if nothing has changed from previous iteration, skip rest - if (nestedCnt === nestedLookupTbl.length) { - for (let i = 0; i < nestedLookupTbl.length; i++) { - const fTblField = - nestedLookupTbl[i].typeOptions.foreignTableRollupColumnId; - const name = aTbl_getColumnName(fTblField); - updateMigrationSkipLog( - ncSchema.tablesById[nestedLookupTbl[i].srcTableId]?.title, - nestedLookupTbl[i].name, - nestedLookupTbl[i].type, - `foreign table field not found [${name.tn}/${name.cn}]` - ); - } - if (enableErrorLogs) - console.log( - `## Failed to configure ${nestedLookupTbl.length} lookups` - ); - break; - } - - // Nested lookup - nestedCnt = nestedLookupTbl.length; - for (let i = 0; i < nestedLookupTbl.length; i++) { - const srcTableId = nestedLookupTbl[0].srcTableId; - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId - ); - - if ( - ncLookupColumnId === undefined || - ncRelationColumnId === undefined - ) { - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - nestedLookupTbl[0].name - ); - - logDetailed( - `Configuring Nested Lookup: Level-${level} [${i + 1}/${nestedCnt} ${ - ncName.title - }]` - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === nestedLookupTbl[0].name - )?.id; - await sMap.addToMappingTbl( - nestedLookupTbl[0].id, - ncId, - nestedLookupTbl[0].name, - ncTbl.id - ); - - // remove entry - nestedLookupTbl.splice(0, 1); - } - level++; - } - } - - function getRollupNcFunction(aTblFunction) { - const fn = aTblFunction.split('(')[0]; - const aTbl_ncRollUp = { - AND: '', - ARRAYCOMPACT: '', - ARRAYJOIN: '', - ARRAYUNIQUE: '', - AVERAGE: 'average', - CONCATENATE: '', - COUNT: 'count', - COUNTA: '', - COUNTALL: '', - MAX: 'max', - MIN: 'min', - OR: '', - SUM: 'sum', - XOR: '', - }; - return aTbl_ncRollUp[fn]; - } - - //@ts-ignore - async function nocoCreateRollup(aTblSchema) { - // Rollup - for (let idx = 0; idx < aTblSchema.length; idx++) { - const aTblColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'rollup' - ); - - // parent table ID - // let srcTableId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const srcTableId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - if (aTblColumns.length) { - // rollup exist - for (let i = 0; i < aTblColumns.length; i++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Rollup :: [${ - i + 1 - }/${aTblColumns.length}] ${aTblSchema[idx].name}` - ); - - // fetch associated rollup function - // skip column creation if supported rollup function does not exist - const ncRollupFn = getRollupNcFunction( - aTblColumns[i].typeOptions.formulaTextParsed - ); - // const ncRollupFn = ''; - - if (ncRollupFn === '' || ncRollupFn === undefined) { - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - `rollup function ${aTblColumns[i].typeOptions.formulaTextParsed} not supported` - ); - continue; - } - - // something is not right, skip - if ( - aTblColumns[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - if (enableErrorLogs) - console.log(`## Invalid column IDs mapped; skip`); - - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'invalid column ID in dependency list' - ); - continue; - } - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.relationColumnId - ); - const ncRollupColumnId = await sMap.getNcIdFromAtId( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - - if (ncRollupColumnId === undefined) { - aTblColumns[i]['srcTableId'] = srcTableId; - nestedRollupTbl.push(aTblColumns[i]); - continue; - } - - // skip, if rollup column was pointing to another virtual column - const ncColSchema = await nc_getColumnSchema( - aTblColumns[i].typeOptions.foreignTableRollupColumnId - ); - if ( - ncColSchema?.uidt === UITypes.Formula || - ncColSchema?.uidt === UITypes.Lookup || - ncColSchema?.uidt === UITypes.Rollup || - ncColSchema?.uidt === UITypes.Checkbox - ) { - updateMigrationSkipLog( - srcTableSchema.title, - aTblColumns[i].name, - aTblColumns[i].type, - 'rollup referring to a column type not supported currently' - ); - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - aTblColumns[i].name - ); - - logDetailed(`NC API: dbTableColumn.create ROLLUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Rollup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_rollup_column_id: ncRollupColumnId, - rollup_function: ncRollupFn, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === aTblColumns[i].name - )?.id; - await sMap.addToMappingTbl( - aTblColumns[i].id, - ncId, - aTblColumns[i].name, - ncTbl.id - ); - } - } - } - logDetailed(`Nested rollup: ${nestedRollupTbl.length}`); - } - - //@ts-ignore - async function nocoLookupForRollup() { - const nestedCnt = nestedLookupTbl.length; - for (let i = 0; i < nestedLookupTbl.length; i++) { - const srcTableId = nestedLookupTbl[0].srcTableId; - const srcTableSchema = ncSchema.tablesById[srcTableId]; - - const ncRelationColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.relationColumnId - ); - const ncLookupColumnId = await sMap.getNcIdFromAtId( - nestedLookupTbl[0].typeOptions.foreignTableRollupColumnId - ); - - if (ncLookupColumnId === undefined || ncRelationColumnId === undefined) { - continue; - } - - const ncName = nc_getSanitizedColumnName( - srcTableSchema, - nestedLookupTbl[0].name - ); - - logDetailed( - `Configuring Lookup over Rollup :: [${i + 1}/${nestedCnt}] ${ - ncName.title - }` - ); - - logDetailed(`NC API: dbTableColumn.create LOOKUP ${ncName.title}`); - const _perfStart = recordPerfStart(); - const ncTbl: any = await api.dbTableColumn.create(srcTableId, { - uidt: UITypes.Lookup, - title: ncName.title, - column_name: ncName.column_name, - fk_relation_column_id: ncRelationColumnId, - fk_lookup_column_id: ncLookupColumnId, - }); - recordPerfStats(_perfStart, 'dbTableColumn.create'); - - updateNcTblSchema(ncTbl); - - const ncId = ncTbl.columns.find( - (x) => x.title === nestedLookupTbl[0].name - )?.id; - await sMap.addToMappingTbl( - nestedLookupTbl[0].id, - ncId, - nestedLookupTbl[0].name, - ncTbl.id - ); - - // remove entry - nestedLookupTbl.splice(0, 1); - } - } - - async function nocoSetPrimary(aTblSchema) { - for (let idx = 0; idx < aTblSchema.length; idx++) { - logDetailed( - `[${idx + 1}/${aTblSchema.length}] Configuring Display value : ${ - aTblSchema[idx].name - }` - ); - - const pColId = aTblSchema[idx].primaryColumnId; - const ncColId = await sMap.getNcIdFromAtId(pColId); - - // skip primary column configuration if we field not migrated - if (ncColId) { - logDetailed(`NC API: dbTableColumn.primaryColumnSet`); - const _perfStart = recordPerfStart(); - await api.dbTableColumn.primaryColumnSet(ncColId); - recordPerfStats(_perfStart, 'dbTableColumn.primaryColumnSet'); - - // update schema - const ncTblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - await updateNcTblSchemaById(ncTblId); - } - } - } - - // retrieve nc-view column ID from corresponding nc-column ID - async function nc_getViewColumnId(viewId, viewType, ncColumnId) { - // retrieve view Info - let viewDetails; - - const _perfStart = recordPerfStart(); - if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.formRead'); - } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.galleryRead'); - } else { - viewDetails = await api.dbView.gridColumnsList(viewId); - recordPerfStats(_perfStart, 'dbView.gridColumnsList'); - } - - return viewDetails.find((x) => x.fk_column_id === ncColumnId)?.id; - } - - ////////// Data processing - - async function nocoBaseDataProcessing_v2(sDB, table, record) { - const recordHash = hash(record); - const rec = { ...record.fields }; - - // kludge - - // trim spaces on either side of column name - // leads to error in NocoDB - Object.keys(rec).forEach((key) => { - const replacedKey = key.trim().replace(/\./g, '_'); - if (key !== replacedKey) { - rec[replacedKey] = rec[key]; - delete rec[key]; - } - }); - - // post-processing on the record - for (const [key, value] of Object.entries(rec as { [key: string]: any })) { - // retrieve datatype - const dt = table.columns.find((x) => x.title === key)?.uidt; - - // always process LTAR, Lookup, and Rollup columns as we delete the key after processing - if ( - !value && - dt !== UITypes.LinkToAnotherRecord && - dt !== UITypes.Lookup && - dt !== UITypes.Rollup - ) { - rec[key] = null; - continue; - } - - switch (dt) { - // https://www.npmjs.com/package/validator - // default value: digits_after_decimal: [2] - // if currency, set decimal place to 2 - // - case UITypes.Currency: - rec[key] = (+value).toFixed(2); - break; - - // we will pick up LTAR once all table data's are in place - case UITypes.LinkToAnotherRecord: - if (storeLinks) { - if (ncLinkDataStore[table.title][record.id] === undefined) - ncLinkDataStore[table.title][record.id] = { - id: record.id, - fields: {}, - }; - ncLinkDataStore[table.title][record.id]['fields'][key] = value; - } - delete rec[key]; - break; - - // these will be automatically populated depending on schema configuration - case UITypes.Lookup: - case UITypes.Rollup: - delete rec[key]; - break; - - case UITypes.Collaborator: - // in case of multi-collaborator, this will be an array - if (Array.isArray(value)) { - let collaborators = ''; - for (let i = 0; i < value.length; i++) { - collaborators += `${value[i]?.name} <${value[i]?.email}>, `; - rec[key] = collaborators; - } - } else rec[key] = `${value?.name} <${value?.email}>`; - break; - - case UITypes.Button: - rec[key] = `${value?.label} <${value?.url}>`; - break; - - case UITypes.DateTime: - case UITypes.CreateTime: - case UITypes.LastModifiedTime: - rec[key] = dayjs(value).format('YYYY-MM-DD HH:mm'); - break; - - case UITypes.Date: - if (/\d{5,}/.test(value)) { - // skip - rec[key] = null; - logBasic(`:: Invalid date ${value}`); - } else { - rec[key] = dayjs(value).format('YYYY-MM-DD'); - } - break; - - case UITypes.SingleSelect: - if (value === '') { - rec[key] = 'nc_empty'; - } - rec[key] = value; - break; - - case UITypes.MultiSelect: - rec[key] = value - ?.map((v) => { - if (v === '') { - return 'nc_empty'; - } - return `${v.replace(/,/g, '.')}`; - }) - .join(','); - break; - - case UITypes.Attachment: - if (!syncDB.options.syncAttachment) rec[key] = null; - else { - let tempArr = []; - - try { - logBasic( - ` :: Retrieving attachment :: ${value - ?.map((a) => a.filename?.split('?')?.[0]) - .join(', ')}` - ); - tempArr = await api.storage.uploadByUrl( - { - path: `noco/${sDB.projectName}/${table.title}/${key}`, - }, - value?.map((attachment) => ({ - fileName: attachment.filename?.split('?')?.[0], - url: attachment.url, - size: attachment.size, - mimetype: attachment.type, - })) - ); - } catch (e) { - console.log(e); - } - - rec[key] = JSON.stringify(tempArr); - } - break; - - case UITypes.SingleLineText: - // Barcode data - if (value?.text) { - rec[key] = value.text; - } - break; - - default: - break; - } - } - - // insert airtable record ID explicitly into each records - rec[ncSysFields.id] = record.id; - rec[ncSysFields.hash] = recordHash; - - return rec; - } - - // @ts-ignore - async function nocoReadDataSelected(projName, table, callback, fields) { - return new Promise((resolve, reject) => { - base(table.title) - .select({ - pageSize: 100, - // maxRecords: 100, - fields: fields, - }) - .eachPage( - async function page(records, fetchNextPage) { - // This function (`page`) will get called for each page of records. - // records.forEach(record => callback(table, record)); - logBasic( - `:: ${table.title} / ${fields} : ${ - recordCnt + 1 - } ~ ${(recordCnt += 100)}` - ); - await Promise.all( - records.map((r) => callback(projName, table, r, fields)) - ); - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - fetchNextPage(); - }, - function done(err) { - if (err) { - console.error(err); - reject(err); - } - resolve(null); - } - ); - }); - } - - ////////// - - function nc_isLinkExists(airtableFieldId) { - return !!ncLinkMappingTable.find( - (x) => x.aTbl.typeOptions.symmetricColumnId === airtableFieldId - ); - } - - async function nocoCreateProject(projName) { - // create empty project (XC-DB) - logDetailed(`Create Project: ${projName}`); - const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.create({ - title: projName, - }); - recordPerfStats(_perfStart, 'project.create'); - } - - async function nocoGetProject(projId) { - // create empty project (XC-DB) - logDetailed(`Getting project meta: ${projId}`); - const _perfStart = recordPerfStart(); - ncCreatedProjectSchema = await api.project.read(projId); - recordPerfStats(_perfStart, 'project.read'); - } - - async function nocoConfigureGalleryView(sDB, aTblSchema) { - if (!sDB.options.syncViews) return; - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = (await nc_getTableSchema(aTblSchema[idx].name)).id; - const galleryViews = aTblSchema[idx].views.filter( - (x) => x.type === 'gallery' - ); - - const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.gallery += galleryViews.length; - - for (let i = 0; i < galleryViews.length; i++) { - logDetailed(` Axios fetch view-data`); - - // create view - await getViewData(galleryViews[i].id); - const viewName = aTblSchema[idx].views.find( - (x) => x.id === galleryViews[i].id - )?.name; - - logBasic( - `:: [${configuredViews + i + 1}/${rtc.view.total}] Gallery : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - logDetailed(`NC API dbView.galleryCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - await api.dbView.galleryCreate(tblId, { title: viewName }); - recordPerfStats(_perfStart, 'dbView.galleryCreate'); - - await updateNcTblSchemaById(tblId); - // syncLog(`[${idx+1}/${aTblSchema.length}][Gallery View][${i+1}/${galleryViews.length}] Create ${viewName}`) - - // await nc_configureFields(g.id, vData, aTblSchema[idx].name, viewName, 'gallery'); - } - } - } - - async function nocoConfigureFormView(sDB, aTblSchema) { - if (!sDB.options.syncViews) return; - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const formViews = aTblSchema[idx].views.filter((x) => x.type === 'form'); - - const configuredViews = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.form += formViews.length; - for (let i = 0; i < formViews.length; i++) { - logDetailed(` Axios fetch view-data`); - - // create view - const vData = await getViewData(formViews[i].id); - const viewName = aTblSchema[idx].views.find( - (x) => x.id === formViews[i].id - )?.name; - - logBasic( - `:: [${configuredViews + i + 1}/${rtc.view.total}] Form : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - // everything is default - let refreshMode = 'NO_REFRESH'; - let msg = 'Thank you for submitting the form!'; - let desc = ''; - - // response will not include form object if everything is default - // - if (vData.metadata?.form) { - if (vData.metadata.form?.refreshAfterSubmit) - refreshMode = vData.metadata.form.refreshAfterSubmit; - if (vData.metadata.form?.afterSubmitMessage) - msg = vData.metadata.form.afterSubmitMessage; - if (vData.metadata.form?.description) - desc = vData.metadata.form.description; - } - - const formData = { - title: viewName, - heading: viewName, - subheading: desc, - success_msg: msg, - submit_another_form: refreshMode.includes('REFRESH_BUTTON'), - show_blank_form: refreshMode.includes('AUTO_REFRESH'), - }; - - logDetailed(`NC API dbView.formCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - const f = await api.dbView.formCreate(tblId, formData); - recordPerfStats(_perfStart, 'dbView.formCreate'); - - logDetailed( - `[${idx + 1}/${aTblSchema.length}][Form View][${i + 1}/${ - formViews.length - }] Create ${viewName}` - ); - - await updateNcTblSchemaById(tblId); - - logDetailed(` Configure show/hide columns`); - await nc_configureFields( - f.id, - vData, - aTblSchema[idx].name, - viewName, - 'form' - ); - } - } - } - - async function nocoConfigureGridView(sDB, aTblSchema) { - for (let idx = 0; idx < aTblSchema.length; idx++) { - const tblId = await sMap.getNcIdFromAtId(aTblSchema[idx].id); - const gridViews = aTblSchema[idx].views.filter((x) => x.type === 'grid'); - - let viewCnt = idx; - if (syncDB.options.syncViews) - viewCnt = rtc.view.grid + rtc.view.gallery + rtc.view.form; - rtc.view.grid += gridViews.length; - - for (let i = 0; i < (sDB.options.syncViews ? gridViews.length : 1); i++) { - logDetailed(` Axios fetch view-data`); - // fetch viewData JSON - const vData = await getViewData(gridViews[i].id); - - // retrieve view name & associated NC-ID - const viewName = aTblSchema[idx].views.find( - (x) => x.id === gridViews[i].id - )?.name; - const _perfStart = recordPerfStart(); - const viewList: any = await api.dbView.list(tblId); - recordPerfStats(_perfStart, 'dbView.list'); - - let ncViewId = viewList?.list?.find((x) => x.tn === viewName)?.id; - - logBasic( - `:: [${viewCnt + i + 1}/${rtc.view.total}] Grid : ${ - aTblSchema[idx].name - } / ${viewName}` - ); - - // create view (default already created) - if (i > 0) { - logDetailed(`NC API dbView.gridCreate :: ${viewName}`); - const _perfStart = recordPerfStart(); - const viewCreated = await api.dbView.gridCreate(tblId, { - title: viewName, - }); - recordPerfStats(_perfStart, 'dbView.gridCreate'); - - await updateNcTblSchemaById(tblId); - await sMap.addToMappingTbl( - gridViews[i].id, - viewCreated.id, - viewName, - tblId - ); - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Create ${viewName}`) - ncViewId = viewCreated.id; - } - - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Hide columns ${viewName}`) - logDetailed(` Configure show/hide columns`); - await nc_configureFields( - ncViewId, - vData, - aTblSchema[idx].name, - viewName, - 'grid' - ); - - // configure filters - if (vData?.filters) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure filters ${viewName}`) - logDetailed(` Configure filter set`); - - // skip filters if nested - if (!vData.filters.filterSet.find((x) => x?.type === 'nested')) { - await nc_configureFilters(ncViewId, vData.filters); - } - } - - // configure sort - if (vData?.lastSortsApplied?.sortSet.length) { - // syncLog(`[${idx+1}/${aTblSchema.length}][Grid View][${i+1}/${gridViews.length}] Configure sort ${viewName}`) - logDetailed(` Configure sort set`); - await nc_configureSort(ncViewId, vData.lastSortsApplied); - } - } - } - } - - async function nocoAddUsers(aTblSchema) { - const userRoles = { - owner: 'owner', - create: 'creator', - edit: 'editor', - comment: 'commenter', - read: 'viewer', - none: 'viewer', - }; - const userList = aTblSchema.appBlanket.userInfoById; - const totalUsers = Object.keys(userList).length; - let cnt = 0; - const insertJobs: Promise[] = []; - - for (const [, value] of Object.entries( - userList as { [key: string]: any } - )) { - logDetailed( - `[${++cnt}/${totalUsers}] NC API auth.projectUserAdd :: ${value.email}` - ); - const _perfStart = recordPerfStart(); - insertJobs.push( - api.auth - .projectUserAdd(ncCreatedProjectSchema.id, { - email: value.email, - roles: userRoles[value.permissionLevel], - }) - .catch((e) => - e.response?.data?.msg - ? logBasic(`NOTICE: ${e.response.data.msg}`) - : console.log(e) - ) - ); - recordPerfStats(_perfStart, 'auth.projectUserAdd'); - } - await Promise.all(insertJobs); - } - - function updateNcTblSchema(tblSchema) { - const tblId = tblSchema.id; - - // replace entry from array if already exists - const idx = ncSchema.tables.findIndex((x) => x.id === tblId); - if (idx !== -1) ncSchema.tables.splice(idx, 1); - ncSchema.tables.push(tblSchema); - - // overwrite object if it exists - ncSchema.tablesById[tblId] = tblSchema; - } - - async function updateNcTblSchemaById(tblId) { - const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(tblId); - recordPerfStats(_perfStart, 'dbTable.read'); - - updateNcTblSchema(ncTbl); - } - - /////////////////////// - - // statistics - // - const migrationStats = []; - - async function generateMigrationStats(aTblSchema) { - const migrationStatsObj = { - table_name: '', - aTbl: { - columns: 0, - links: 0, - lookup: 0, - rollup: 0, - }, - nc: { - columns: 0, - links: 0, - lookup: 0, - rollup: 0, - invalidColumn: 0, - }, - }; - for (let idx = 0; idx < aTblSchema.length; idx++) { - migrationStatsObj.table_name = aTblSchema[idx].name; - - const aTblLinkColumns = aTblSchema[idx].columns.filter( - (x) => x.type === 'foreignKey' - ); - const aTblLookup = aTblSchema[idx].columns.filter( - (x) => x.type === 'lookup' - ); - const aTblRollup = aTblSchema[idx].columns.filter( - (x) => x.type === 'rollup' - ); - - let invalidColumnId = 0; - for (let i = 0; i < aTblLookup.length; i++) { - if ( - aTblLookup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - invalidColumnId++; - } - } - for (let i = 0; i < aTblRollup.length; i++) { - if ( - aTblRollup[i]?.typeOptions?.dependencies?.invalidColumnIds?.length - ) { - invalidColumnId++; - } - } - - migrationStatsObj.aTbl.columns = aTblSchema[idx].columns.length; - migrationStatsObj.aTbl.links = aTblLinkColumns.length; - migrationStatsObj.aTbl.lookup = aTblLookup.length; - migrationStatsObj.aTbl.rollup = aTblRollup.length; - - const ncTbl = await nc_getTableSchema(aTblSchema[idx].name); - const linkColumn = ncTbl.columns.filter( - (x) => x.uidt === UITypes.LinkToAnotherRecord - ); - const lookup = ncTbl.columns.filter((x) => x.uidt === UITypes.Lookup); - const rollup = ncTbl.columns.filter((x) => x.uidt === UITypes.Rollup); - - // all links hardwired as m2m. m2m generates additional tables per link - // hence link/2 - migrationStatsObj.nc.columns = - ncTbl.columns.length - linkColumn.length / 2; - migrationStatsObj.nc.links = linkColumn.length / 2; - migrationStatsObj.nc.lookup = lookup.length; - migrationStatsObj.nc.rollup = rollup.length; - migrationStatsObj.nc.invalidColumn = invalidColumnId; - - const temp = JSON.parse(JSON.stringify(migrationStatsObj)); - migrationStats.push(temp); - } - - const columnSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.columns; - }, 0); - const linkSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.links; - }, 0); - const lookupSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.lookup; - }, 0); - const rollupSum = migrationStats.reduce((accumulator, object) => { - return accumulator + object.nc.rollup; - }, 0); - - logBasic(`Quick Summary:`); - logBasic(`:: Total Tables: ${aTblSchema.length}`); - logBasic(`:: Total Columns: ${columnSum}`); - logBasic(`:: Links: ${linkSum}`); - logBasic(`:: Lookup: ${lookupSum}`); - logBasic(`:: Rollup: ${rollupSum}`); - logBasic(`:: Total Filters: ${rtc.filter}`); - logBasic(`:: Total Sort: ${rtc.sort}`); - logBasic(`:: Total Views: ${rtc.view.total}`); - logBasic(`:: Grid: ${rtc.view.grid}`); - logBasic(`:: Gallery: ${rtc.view.gallery}`); - logBasic(`:: Form: ${rtc.view.form}`); - logBasic(`:: Total Records: ${rtc.data.records}`); - logBasic(`:: Total Nested Links: ${rtc.data.nestedLinks}`); - - const duration = Date.now() - start; - logBasic(`:: Migration time: ${duration}`); - logBasic(`:: Axios fetch count: ${rtc.fetchAt.count}`); - logBasic(`:: Axios fetch time: ${rtc.fetchAt.time}`); - - if (debugMode) { - await writeJsonFileAsync('stats.json', perfStats, { spaces: 2 }); - const perflog = []; - for (let i = 0; i < perfStats.length; i++) { - perflog.push(`${perfStats[i].e}, ${perfStats[i].d}`); - } - await writeJsonFileAsync('stats.csv', perflog, { spaces: 2 }); - await writeJsonFileAsync('skip.txt', rtc.migrationSkipLog.log, { - spaces: 2, - }); - } - - T.event({ - event: 'a:airtable-import:success', - data: { - stats: { - migrationTime: duration, - totalTables: aTblSchema.length, - totalColumns: columnSum, - links: linkSum, - lookup: lookupSum, - rollup: rollupSum, - totalFilters: rtc.filter, - totalSort: rtc.sort, - view: { - total: rtc.view.total, - grid: rtc.view.grid, - gallery: rtc.view.gallery, - form: rtc.view.form, - }, - axios: { - count: rtc.fetchAt.count, - time: rtc.fetchAt.time, - }, - totalRecords: rtc.data.records, - nestedLinks: rtc.data.nestedLinks, - }, - }, - }); - } - - ////////////////////////////// - // filters - - const filterMap = { - '=': 'eq', - '!=': 'neq', - '<': 'lt', - '<=': 'lte', - '>': 'gt', - '>=': 'gte', - isEmpty: 'empty', - isNotEmpty: 'notempty', - isWithin: 'isWithin', - contains: 'like', - doesNotContain: 'nlike', - isAnyOf: 'anyof', - isNoneOf: 'nanyof', - '|': 'anyof', - '&': 'allof', - }; - - async function nc_configureFilters(viewId, f) { - for (let i = 0; i < f.filterSet.length; i++) { - const filter = f.filterSet[i]; - const colSchema = await nc_getColumnSchema(filter.columnId); - - // column not available; - // one of not migrated column; - if (!colSchema) { - updateMigrationSkipLog( - await sMap.getNcNameFromAtId(viewId), - colSchema.title, - colSchema.uidt, - `filter config skipped; column not migrated` - ); - continue; - } - const columnId = colSchema.id; - const datatype = colSchema.uidt; - const ncFilters = []; - - if (datatype === UITypes.Date || datatype === UITypes.DateTime) { - let comparison_op = null; - let comparison_sub_op = null; - let value = null; - if (['isEmpty', 'isNotEmpty'].includes(filter.operator)) { - comparison_op = filter.operator === 'isEmpty' ? 'blank' : 'notblank'; - } else { - if ('numberOfDays' in filter.value) { - value = filter.value['numberOfDays']; - } else if ('exactDate' in filter.value) { - value = filter.value['exactDate']; - } - comparison_op = filterMap[filter.operator]; - comparison_sub_op = filter.value.mode; - } - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op, - comparison_sub_op, - value, - }; - ncFilters.push(fx); - } - - // single-select & multi-select - else if ( - datatype === UITypes.SingleSelect || - datatype === UITypes.MultiSelect - ) { - if (filter.operator === 'doesNotContain') { - filter.operator = 'isNoneOf'; - } - // if array, break it down to multiple filters - if (Array.isArray(filter.value)) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: ( - await Promise.all( - filter.value.map(async (f) => await sMap.getNcNameFromAtId(f)) - ) - ).join(','), - }; - ncFilters.push(fx); - } - // not array - add as is - else if (filter.value) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: await sMap.getNcNameFromAtId(filter.value), - }; - ncFilters.push(fx); - } - } - - // other data types (number/ text/ long text/ ..) - else if (filter.value) { - const fx = { - fk_column_id: columnId, - logical_op: f.conjunction, - comparison_op: filterMap[filter.operator], - value: filter.value, - }; - ncFilters.push(fx); - } - - // insert filters - for (let i = 0; i < ncFilters.length; i++) { - const _perfStart = recordPerfStart(); - await api.dbTableFilter.create(viewId, { - ...ncFilters[i], - }); - recordPerfStats(_perfStart, 'dbTableFilter.create'); - - rtc.filter++; - } - } - } - - async function nc_configureSort(viewId, s) { - for (let i = 0; i < s.sortSet.length; i++) { - const columnId = (await nc_getColumnSchema(s.sortSet[i].columnId))?.id; - - if (columnId) { - const _perfStart = recordPerfStart(); - await api.dbTableSort.create(viewId, { - fk_column_id: columnId, - direction: s.sortSet[i].ascending ? 'asc' : 'dsc', - }); - recordPerfStats(_perfStart, 'dbTableSort.create'); - } - rtc.sort++; - } - } - - async function nc_configureFields(_viewId, _c, tblName, viewName, viewType?) { - // force hide PK column - const hiddenColumns = [ncSysFields.id, ncSysFields.hash]; - const c = _c.columnOrder; - - // column order corrections - // retrieve table schema - const ncTbl = await nc_getTableSchema(tblName); - // retrieve view ID - const viewId = ncTbl.views.find((x) => x.title === viewName).id; - let viewDetails; - - const _perfStart = recordPerfStart(); - if (viewType === 'form') { - viewDetails = (await api.dbView.formRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.formRead'); - } else if (viewType === 'gallery') { - viewDetails = (await api.dbView.galleryRead(viewId)).columns; - recordPerfStats(_perfStart, 'dbView.galleryRead'); - } else { - viewDetails = await api.dbView.gridColumnsList(viewId); - recordPerfStats(_perfStart, 'dbView.gridColumnsList'); - } - - // nc-specific columns; default hide. - for (let j = 0; j < hiddenColumns.length; j++) { - const ncColumnId = ncTbl.columns.find( - (x) => x.title === hiddenColumns[j] - ).id; - const ncViewColumnId = viewDetails.find( - (x) => x.fk_column_id === ncColumnId - )?.id; - // const ncViewColumnId = await nc_getViewColumnId( - // viewId, - // viewType, - // ncColumnId - // ); - if (ncViewColumnId === undefined) continue; - - // first two positions held by record id & record hash - const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, { - show: false, - order: j + 1 + c.length, - }); - recordPerfStats(_perfStart, 'dbViewColumn.update'); - } - - // rest of the columns from airtable- retain order & visibility property - for (let j = 0; j < c.length; j++) { - const ncColumnId = await sMap.getNcIdFromAtId(c[j].columnId); - const ncViewColumnId = await nc_getViewColumnId( - viewId, - viewType, - ncColumnId - ); - if (ncViewColumnId === undefined) continue; - - // first two positions held by record id & record hash - const configData = { show: c[j].visibility, order: j + 1 }; - if (viewType === 'form') { - if (_c?.metadata?.form?.fieldsByColumnId?.[c[j].columnId]) { - const x = _c.metadata.form.fieldsByColumnId[c[j].columnId]; - const formData = { ...configData }; - if (x?.title) formData[`label`] = x.title; - if (x?.required) formData[`required`] = x.required; - if (x?.description) formData[`description`] = x.description; - const _perfStart = recordPerfStart(); - await api.dbView.formColumnUpdate(ncViewColumnId, formData); - recordPerfStats(_perfStart, 'dbView.formColumnUpdate'); - } - } - const _perfStart = recordPerfStart(); - await api.dbViewColumn.update(viewId, ncViewColumnId, configData); - recordPerfStats(_perfStart, 'dbViewColumn.update'); - } - } - - /////////////////////////////////////////////////////////////////////////////// - let recordCnt = 0; - try { - logBasic('SDK initialized'); - api = new Api({ - baseURL: syncDB.baseURL, - headers: { - 'xc-auth': syncDB.authToken, - }, - }); - - logDetailed('Project initialization started'); - // delete project if already exists - if (debugMode) await init(syncDB); - - logDetailed('Project initialized'); - - logBasic('Retrieving Airtable schema'); - // read schema file - const schema = await getAirtableSchema(syncDB); - const aTblSchema = schema.tableSchemas; - logDetailed('Project schema extraction completed'); - - if (!syncDB.projectId) { - if (!syncDB.projectName) - throw new Error('Project name or id not provided'); - // create empty project - await nocoCreateProject(syncDB.projectName); - logDetailed('Project created'); - } else { - await nocoGetProject(syncDB.projectId); - syncDB.projectName = ncCreatedProjectSchema?.title; - syncDB.baseId = syncDB.baseId || ncCreatedProjectSchema.bases[0].id; - logDetailed('Getting existing project meta'); - } - - logBasic('Importing Tables...'); - // prepare table schema (base) - await nocoCreateBaseSchema(aTblSchema); - logDetailed('Table creation completed'); - - logDetailed('Configuring Links'); - // add LTAR - await nocoCreateLinkToAnotherRecord(aTblSchema); - logDetailed('Migrating LTAR columns completed'); - - if (syncDB.options.syncLookup) { - logDetailed(`Configuring Lookup`); - // add look-ups - await nocoCreateLookups(aTblSchema); - logDetailed('Migrating Lookup columns completed'); - } - - if (syncDB.options.syncRollup) { - logDetailed('Configuring Rollup'); - // add roll-ups - await nocoCreateRollup(aTblSchema); - logDetailed('Migrating Rollup columns completed'); - - if (syncDB.options.syncLookup) { - logDetailed('Migrating Lookup form Rollup columns'); - // lookups for rollup - await nocoLookupForRollup(); - logDetailed('Migrating Lookup form Rollup columns completed'); - } - } - logDetailed('Configuring Display Value column'); - // configure Display Value - await nocoSetPrimary(aTblSchema); - logDetailed('Configuring Display Value column completed'); - - logBasic('Configuring User(s)'); - // add users - await nocoAddUsers(schema); - logDetailed('Adding users completed'); - - // hide-fields - // await nocoReconfigureFields(aTblSchema); - - logBasic('Syncing views'); - // configure views - await nocoConfigureGridView(syncDB, aTblSchema); - await nocoConfigureFormView(syncDB, aTblSchema); - await nocoConfigureGalleryView(syncDB, aTblSchema); - logDetailed('Syncing views completed'); - - if (syncDB.options.syncData) { - try { - // await nc_DumpTableSchema(); - const _perfStart = recordPerfStart(); - const ncTblList = await api.base.tableList( - ncCreatedProjectSchema.id, - syncDB.baseId - ); - recordPerfStats(_perfStart, 'base.tableList'); - - logBasic('Reading Records...'); - - const recordsMap = {}; - - for (let i = 0; i < ncTblList.list.length; i++) { - // not a migrated table, skip - if ( - undefined === - aTblSchema.find((x) => x.name === ncTblList.list[i].title) - ) - continue; - - const _perfStart = recordPerfStart(); - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - recordPerfStats(_perfStart, 'dbTable.read'); - - recordCnt = 0; - // await nocoReadData(syncDB, ncTbl); - - recordsMap[ncTbl.id] = await importData({ - projectName: syncDB.projectName, - table: ncTbl, - base, - api, - logBasic, - nocoBaseDataProcessing_v2, - sDB: syncDB, - logDetailed, - }); - rtc.data.records += await recordsMap[ncTbl.id].getCount(); - - logDetailed(`Data inserted from ${ncTbl.title}`); - } - - logBasic('Configuring Record Links...'); - for (let i = 0; i < ncTblList.list.length; i++) { - // not a migrated table, skip - if ( - undefined === - aTblSchema.find((x) => x.name === ncTblList.list[i].title) - ) - continue; - - const ncTbl = await api.dbTable.read(ncTblList.list[i].id); - - rtc.data.nestedLinks += await importLTARData({ - table: ncTbl, - projectName: syncDB.projectName, - api, - base, - fields: null, //Object.values(tblLinkGroup).flat(), - logBasic, - insertedAssocRef, - logDetailed, - records: recordsMap[ncTbl.id], - atNcAliasRef, - ncLinkMappingTable, - }); - } - - if (storeLinks) { - // const insertJobs: Promise[] = []; - // for (const [pTitle, v] of Object.entries(ncLinkDataStore)) { - // logBasic(`:: ${pTitle}`); - // for (const [, record] of Object.entries(v)) { - // const tbl = ncTblList.list.find(a => a.title === pTitle); - // await nocoLinkProcessing(syncDB.projectName, tbl, record, 0); - // // insertJobs.push( - // // nocoLinkProcessing(syncDB.projectName, tbl, record, 0) - // // ); - // } - // } - // await Promise.all(insertJobs); - // await nocoLinkProcessing(syncDB.projectName, 0, 0, 0); - } else { - // // create link groups (table: link fields) - // // const tblLinkGroup = {}; - // // for (let idx = 0; idx < ncLinkMappingTable.length; idx++) { - // // const x = ncLinkMappingTable[idx]; - // // if (tblLinkGroup[x.aTbl.tblId] === undefined) - // // tblLinkGroup[x.aTbl.tblId] = [x.aTbl.name]; - // // else tblLinkGroup[x.aTbl.tblId].push(x.aTbl.name); - // // } - // // - // // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // // - // // await importLTARData({ - // // table: ncTbl, - // // projectName: syncDB.projectName, - // // api, - // // base, - // // fields: Object.values(tblLinkGroup).flat(), - // // logBasic - // // }); - // for (const [k, v] of Object.entries(tblLinkGroup)) { - // const ncTbl = await nc_getTableSchema(aTbl_getTableName(k).tn); - // - // // not a migrated table, skip - // if (undefined === aTblSchema.find(x => x.name === ncTbl.title)) - // continue; - // - // recordCnt = 0; - // await nocoReadDataSelected( - // syncDB.projectName, - // ncTbl, - // async (projName, table, record, _field) => { - // await nocoLinkProcessing(projName, table, record, _field); - // }, - // v - // ); - // } - } - } catch (error) { - logDetailed( - `There was an error while migrating data! Please make sure your API key (${syncDB.apiKey}) is correct.` - ); - logDetailed(`Error: ${error}`); - } - } - if (generate_migrationStats) { - await generateMigrationStats(aTblSchema); - } - } catch (e) { - if (e.response?.data?.msg) { - T.event({ - event: 'a:airtable-import:error', - data: { error: e.response.data.msg }, - }); - throw new Error(e.response.data.msg); - } - throw e; - } -}; - -export function getUniqueNameGenerator(defaultName = 'name') { - const namesRef = {}; - - return (initName: string = defaultName): string => { - let name = initName === '_' ? defaultName : initName; - let c = 0; - while (name in namesRef) { - name = `${initName}_${++c}`; - } - namesRef[name] = true; - return name; - }; -} - -export interface AirtableSyncConfig { - id: string; - baseURL: string; - authToken: string; - projectName?: string; - projectId?: string; - baseId?: string; - apiKey: string; - shareId: string; - options: { - syncViews: boolean; - syncData: boolean; - syncRollup: boolean; - syncLookup: boolean; - syncFormula: boolean; - syncAttachment: boolean; - }; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts b/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts deleted file mode 100644 index ad004205d1..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/readAndProcessData.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { AirtableBase } from 'airtable/lib/airtable_base'; -import { Api, RelationTypes, TableType, UITypes } from 'nocodb-sdk'; -import EntityMap from './EntityMap'; - -const BULK_DATA_BATCH_SIZE = 500; -const ASSOC_BULK_DATA_BATCH_SIZE = 1000; -const BULK_PARALLEL_PROCESS = 5; - -async function readAllData({ - table, - fields, - base, - logBasic = (_str) => {}, -}: { - table: { title?: string }; - fields?; - base: AirtableBase; - logBasic?: (string) => void; - logDetailed?: (string) => void; -}): Promise { - return new Promise((resolve, reject) => { - let data = null; - - const selectParams: any = { - pageSize: 100, - }; - - if (fields) selectParams.fields = fields; - - base(table.title) - .select(selectParams) - .eachPage( - async function page(records, fetchNextPage) { - if (!data) { - data = new EntityMap(); - await data.init(); - } - - for await (const record of records) { - await data.addRow({ id: record.id, ...record.fields }); - } - - const tmpLength = await data.getCount(); - - logBasic( - `:: Reading '${table.title}' data :: ${Math.max( - 1, - tmpLength - records.length - )} - ${tmpLength}` - ); - - // To fetch the next page of records, call `fetchNextPage`. - // If there are more records, `page` will get called again. - // If there are no more records, `done` will get called. - fetchNextPage(); - }, - async function done(err) { - if (err) { - console.error(err); - return reject(err); - } - resolve(data); - } - ); - }); -} - -export async function importData({ - projectName, - table, - base, - api, - nocoBaseDataProcessing_v2, - sDB, - logDetailed = (_str) => {}, - logBasic = (_str) => {}, -}: { - projectName: string; - table: { title?: string; id?: string }; - fields?; - base: AirtableBase; - logBasic: (string) => void; - logDetailed: (string) => void; - api: Api; - nocoBaseDataProcessing_v2; - sDB; -}): Promise { - try { - // @ts-ignore - const records = await readAllData({ - table, - base, - logDetailed, - logBasic, - }); - - await new Promise(async (resolve) => { - const readable = records.getStream(); - const allRecordsCount = await records.getCount(); - const promises = []; - let tempData = []; - let importedCount = 0; - let activeProcess = 0; - readable.on('data', async (record) => { - promises.push( - new Promise(async (resolve) => { - activeProcess++; - if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); - const { id: rid, ...fields } = record; - const r = await nocoBaseDataProcessing_v2(sDB, table, { - id: rid, - fields, - }); - tempData.push(r); - - if (tempData.length >= BULK_DATA_BATCH_SIZE) { - let insertArray = tempData.splice(0, tempData.length); - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - insertArray - ); - logBasic( - `:: Importing '${ - table.title - }' data :: ${importedCount} - ${Math.min( - importedCount + BULK_DATA_BATCH_SIZE, - allRecordsCount - )}` - ); - importedCount += insertArray.length; - insertArray = []; - } - activeProcess--; - if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); - resolve(true); - }) - ); - }); - readable.on('end', async () => { - await Promise.all(promises); - if (tempData.length > 0) { - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - table.id, - tempData - ); - logBasic( - `:: Importing '${ - table.title - }' data :: ${importedCount} - ${Math.min( - importedCount + BULK_DATA_BATCH_SIZE, - allRecordsCount - )}` - ); - importedCount += tempData.length; - tempData = []; - } - resolve(true); - }); - }); - - return records; - } catch (e) { - console.log(e); - return null; - } -} - -export async function importLTARData({ - table, - fields, - base, - api, - projectName, - insertedAssocRef = {}, - logDetailed = (_str) => {}, - logBasic = (_str) => {}, - records, - atNcAliasRef, - ncLinkMappingTable, -}: { - projectName: string; - table: { title?: string; id?: string }; - fields; - base: AirtableBase; - logDetailed: (string) => void; - logBasic: (string) => void; - api: Api; - insertedAssocRef: { [assocTableId: string]: boolean }; - records?: EntityMap; - atNcAliasRef: { - [ncTableId: string]: { - [ncTitle: string]: string; - }; - }; - ncLinkMappingTable: Record>[]; -}) { - const assocTableMetas: Array<{ - modelMeta: { id?: string; title?: string }; - colMeta: { title?: string }; - curCol: { title?: string }; - refCol: { title?: string }; - }> = []; - const allData = - records || - (await readAllData({ - table, - fields, - base, - logDetailed, - logBasic, - })); - - const modelMeta: any = await api.dbTable.read(table.id); - - for (const colMeta of modelMeta.columns) { - // skip columns which are not LTAR and Many to many - if ( - colMeta.uidt !== UITypes.LinkToAnotherRecord || - colMeta.colOptions.type !== RelationTypes.MANY_TO_MANY - ) { - continue; - } - - // skip if already inserted - if (colMeta.colOptions.fk_mm_model_id in insertedAssocRef) continue; - - // self links: skip if the column under consideration is the add-on column NocoDB creates - if (ncLinkMappingTable.every((a) => a.nc.title !== colMeta.title)) continue; - - // mark as inserted - insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true; - - const assocModelMeta: TableType = (await api.dbTable.read( - colMeta.colOptions.fk_mm_model_id - )) as any; - - // extract associative table and columns meta - assocTableMetas.push({ - modelMeta: assocModelMeta, - colMeta, - curCol: assocModelMeta.columns.find( - (c) => c.id === colMeta.colOptions.fk_mm_child_column_id - ), - refCol: assocModelMeta.columns.find( - (c) => c.id === colMeta.colOptions.fk_mm_parent_column_id - ), - }); - } - - let nestedLinkCnt = 0; - // Iterate over all related M2M associative table - for await (const assocMeta of assocTableMetas) { - let assocTableData = []; - let importedCount = 0; - - // extract insert data from records - await new Promise((resolve) => { - const promises = []; - const readable = allData.getStream(); - let activeProcess = 0; - readable.on('data', async (record) => { - promises.push( - new Promise(async (resolve) => { - activeProcess++; - if (activeProcess >= BULK_PARALLEL_PROCESS) readable.pause(); - const { id: _atId, ...rec } = record; - - // todo: use actual alias instead of sanitized - assocTableData.push( - ...( - rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || [] - ).map((id) => ({ - [assocMeta.curCol.title]: record.id, - [assocMeta.refCol.title]: id, - })) - ); - - if (assocTableData.length >= ASSOC_BULK_DATA_BATCH_SIZE) { - let insertArray = assocTableData.splice(0, assocTableData.length); - logBasic( - `:: Importing '${ - table.title - }' LTAR data :: ${importedCount} - ${Math.min( - importedCount + ASSOC_BULK_DATA_BATCH_SIZE, - insertArray.length - )}` - ); - - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - insertArray - ); - - importedCount += insertArray.length; - insertArray = []; - } - activeProcess--; - if (activeProcess < BULK_PARALLEL_PROCESS) readable.resume(); - resolve(true); - }) - ); - }); - readable.on('end', async () => { - await Promise.all(promises); - if (assocTableData.length >= 0) { - logBasic( - `:: Importing '${ - table.title - }' LTAR data :: ${importedCount} - ${Math.min( - importedCount + ASSOC_BULK_DATA_BATCH_SIZE, - assocTableData.length - )}` - ); - - await api.dbTableRow.bulkCreate( - 'nc', - projectName, - assocMeta.modelMeta.id, - assocTableData - ); - - importedCount += assocTableData.length; - assocTableData = []; - } - resolve(true); - }); - }); - - nestedLinkCnt += importedCount; - } - return nestedLinkCnt; -} diff --git a/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts b/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts deleted file mode 100644 index 7855bbaa2c..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/helpers/syncMap.ts +++ /dev/null @@ -1,31 +0,0 @@ -export const mapTbl = {}; - -// static mapping records between aTblId && ncId -export const addToMappingTbl = function addToMappingTbl( - aTblId, - ncId, - ncName, - parent? -) { - mapTbl[aTblId] = { - ncId: ncId, - ncParent: parent, - // name added to assist in quick debug - ncName: ncName, - }; -}; - -// get NcID from airtable ID -export const getNcIdFromAtId = function getNcIdFromAtId(aId) { - return mapTbl[aId]?.ncId; -}; - -// get nc Parent from airtable ID -export const getNcParentFromAtId = function getNcParentFromAtId(aId) { - return mapTbl[aId]?.ncParent; -}; - -// get nc-title from airtable ID -export const getNcNameFromAtId = function getNcNameFromAtId(aId) { - return mapTbl[aId]?.ncName; -}; diff --git a/packages/nocodb/src/lib/controllers/sync/importApis.ts b/packages/nocodb/src/lib/controllers/sync/importApis.ts deleted file mode 100644 index 029caa0657..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/importApis.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Request, Router } from 'express'; -// import { Queue } from 'bullmq'; -// import axios from 'axios'; -import catchError, { NcError } from '../../meta/helpers/catchError'; -import { Server } from 'socket.io'; -import NocoJobs from '../../jobs/NocoJobs'; -import job, { AirtableSyncConfig } from './helpers/job'; -import SyncSource from '../../models/SyncSource'; -import Noco from '../../Noco'; -import { genJwt } from '../userApi/helpers'; -const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB'; -const AIRTABLE_PROGRESS_JOB = 'AIRTABLE_PROGRESS_JOB'; - -enum SyncStatus { - PROGRESS = 'PROGRESS', - COMPLETED = 'COMPLETED', - FAILED = 'FAILED', -} - -export default ( - router: Router, - sv: Server, - jobs: { [id: string]: { last_message: any } } -) => { - // add importer job handler and progress notification job handler - NocoJobs.jobsMgr.addJobWorker(AIRTABLE_IMPORT_JOB, job); - NocoJobs.jobsMgr.addJobWorker( - AIRTABLE_PROGRESS_JOB, - ({ payload, progress }) => { - sv.to(payload?.id).emit('progress', { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }); - - if (payload?.id in jobs) { - jobs[payload?.id].last_message = { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }; - } - } - ); - - NocoJobs.jobsMgr.addProgressCbk(AIRTABLE_IMPORT_JOB, (payload, progress) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: progress?.msg, - level: progress?.level, - status: progress?.status, - }, - }); - }); - NocoJobs.jobsMgr.addSuccessCbk(AIRTABLE_IMPORT_JOB, (payload) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: 'Complete!', - status: SyncStatus.COMPLETED, - }, - }); - delete jobs[payload?.id]; - }); - NocoJobs.jobsMgr.addFailureCbk(AIRTABLE_IMPORT_JOB, (payload, error: any) => { - NocoJobs.jobsMgr.add(AIRTABLE_PROGRESS_JOB, { - payload, - progress: { - msg: error?.message || 'Failed due to some internal error', - status: SyncStatus.FAILED, - }, - }); - delete jobs[payload?.id]; - }); - - router.post( - '/api/v1/db/meta/import/airtable', - catchError((req, res) => { - NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { - id: req.query.id, - ...req.body, - }); - res.json({}); - }) - ); - router.post( - '/api/v1/db/meta/syncs/:syncId/trigger', - catchError(async (req: Request, res) => { - if (req.params.syncId in jobs) { - NcError.badRequest('Sync already in progress'); - } - - const syncSource = await SyncSource.get(req.params.syncId); - - const user = await syncSource.getUser(); - const token = genJwt(user, Noco.getConfig()); - - // Treat default baseUrl as siteUrl from req object - let baseURL = (req as any).ncSiteUrl; - - // if environment value avail use it - // or if it's docker construct using `PORT` - if (process.env.NC_BASEURL_INTERNAL) { - baseURL = process.env.NC_BASEURL_INTERNAL; - } else if (process.env.NC_DOCKER) { - baseURL = `http://localhost:${process.env.PORT || 8080}`; - } - - setTimeout(() => { - NocoJobs.jobsMgr.add(AIRTABLE_IMPORT_JOB, { - id: req.params.syncId, - ...(syncSource?.details || {}), - projectId: syncSource.project_id, - baseId: syncSource.base_id, - authToken: token, - baseURL, - }); - }, 1000); - - jobs[req.params.syncId] = { - last_message: { - msg: 'Sync started', - }, - }; - res.json({}); - }) - ); - router.post( - '/api/v1/db/meta/syncs/:syncId/abort', - catchError(async (req: Request, res) => { - if (req.params.syncId in jobs) { - delete jobs[req.params.syncId]; - } - res.json({}); - }) - ); -}; diff --git a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts b/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts deleted file mode 100644 index 9da8d70a70..0000000000 --- a/packages/nocodb/src/lib/controllers/sync/syncSourceApis.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Request, Response, Router } from 'express'; - -import SyncSource from '../../../models/SyncSource'; -import { T } from 'nc-help'; -import { PagedResponseImpl } from '../../helpers/PagedResponse'; -import ncMetaAclMw from '../../helpers/ncMetaAclMw'; -import Project from '../../../models/Project'; - -export async function syncSourceList(req: Request, res: Response) { - // todo: pagination - res.json( - new PagedResponseImpl( - await SyncSource.list(req.params.projectId, req.params.baseId) - ) - ); -} - -export async function syncCreate(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:created' }); - const project = await Project.getWithInfo(req.params.projectId); - - const sync = await SyncSource.insert({ - ...req.body, - fk_user_id: (req as any).user.id, - base_id: req.params.baseId ? req.params.baseId : project.bases[0].id, - project_id: req.params.projectId, - }); - res.json(sync); -} - -export async function syncDelete(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:deleted' }); - res.json(await SyncSource.delete(req.params.syncId)); -} - -export async function syncUpdate(req: Request, res: Response) { - T.emit('evt', { evt_type: 'webhooks:updated' }); - - res.json(await SyncSource.update(req.params.syncId, req.body)); -} - -const router = Router({ mergeParams: true }); - -router.get( - '/api/v1/db/meta/projects/:projectId/syncs', - ncMetaAclMw(syncSourceList, 'syncSourceList') -); -router.post( - '/api/v1/db/meta/projects/:projectId/syncs', - ncMetaAclMw(syncCreate, 'syncSourceCreate') -); -router.get( - '/api/v1/db/meta/projects/:projectId/syncs/:baseId', - ncMetaAclMw(syncSourceList, 'syncSourceList') -); -router.post( - '/api/v1/db/meta/projects/:projectId/syncs/:baseId', - ncMetaAclMw(syncCreate, 'syncSourceCreate') -); -router.delete( - '/api/v1/db/meta/syncs/:syncId', - ncMetaAclMw(syncDelete, 'syncSourceDelete') -); -router.patch( - '/api/v1/db/meta/syncs/:syncId', - ncMetaAclMw(syncUpdate, 'syncSourceUpdate') -); - -export default router; diff --git a/packages/nocodb/src/lib/controllers/tableController.ts b/packages/nocodb/src/lib/controllers/tableController.ts index cc8ace772b..d8987ef3cb 100644 --- a/packages/nocodb/src/lib/controllers/tableController.ts +++ b/packages/nocodb/src/lib/controllers/tableController.ts @@ -1,7 +1,6 @@ import { Request, Response, Router } from 'express'; import { TableListType, TableReqType, TableType } from 'nocodb-sdk'; import { getAjvValidatorMw } from '../meta/api/helpers'; -import { tableUpdate } from '../meta/api/tableApis'; import { metaApiMetrics } from '../meta/helpers/apiMetrics'; import ncMetaAclMw from '../meta/helpers/ncMetaAclMw'; import { PagedResponseImpl } from '../meta/helpers/PagedResponse'; @@ -59,6 +58,17 @@ export async function tableReorder(req: Request, res: Response) { ); } + + +export async function tableUpdate(req: Request, res) { + tableService.tableUpdate({ + tableId: req.params.tableId, + table: req.body, + }) + + res.json({ msg: 'success' }) +} + const router = Router({ mergeParams: true }); router.get( '/api/v1/db/meta/projects/:projectId/tables', diff --git a/packages/nocodb/src/lib/controllers/userApi/helpers.ts b/packages/nocodb/src/lib/controllers/userApi/helpers.ts deleted file mode 100644 index 387b51170e..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/helpers.ts +++ /dev/null @@ -1,23 +0,0 @@ -import * as jwt from 'jsonwebtoken'; -import crypto from 'crypto'; -import User from '../../models/User'; -import { NcConfig } from '../../../interface/config'; - -export function genJwt(user: User, config: NcConfig) { - return jwt.sign( - { - email: user.email, - firstname: user.firstname, - lastname: user.lastname, - id: user.id, - roles: user.roles, - token_version: user.token_version, - }, - config.auth.jwt.secret, - config.auth.jwt.options - ); -} - -export function randomTokenString(): string { - return crypto.randomBytes(40).toString('hex'); -} diff --git a/packages/nocodb/src/lib/controllers/userApi/index.ts b/packages/nocodb/src/lib/controllers/userApi/index.ts deleted file mode 100644 index c4a5003430..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './userApis'; diff --git a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts b/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts deleted file mode 100644 index 185fd155ae..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/initAdminFromEnv.ts +++ /dev/null @@ -1,283 +0,0 @@ -import User from '../../../models/User'; -import { v4 as uuidv4 } from 'uuid'; -import { promisify } from 'util'; - -import bcrypt from 'bcryptjs'; -import Noco from '../../../Noco'; -import { CacheScope, MetaTable } from '../../../utils/globals'; -import ProjectUser from '../../../models/ProjectUser'; -import { validatePassword } from 'nocodb-sdk'; -import boxen from 'boxen'; -import NocoCache from '../../../cache/NocoCache'; -import { T } from 'nc-help'; - -const { isEmail } = require('validator'); -const rolesLevel = { owner: 0, creator: 1, editor: 2, commenter: 3, viewer: 4 }; - -export default async function initAdminFromEnv(_ncMeta = Noco.ncMeta) { - if (process.env.NC_ADMIN_EMAIL && process.env.NC_ADMIN_PASSWORD) { - if (!isEmail(process.env.NC_ADMIN_EMAIL?.trim())) { - console.log( - '\n', - boxen( - `Provided admin email '${process.env.NC_ADMIN_EMAIL}' is not valid`, - { - title: 'Invalid admin email', - padding: 1, - borderStyle: 'double', - titleAlignment: 'center', - borderColor: 'red', - } - ), - '\n' - ); - process.exit(1); - } - - const { valid, error, hint } = validatePassword( - process.env.NC_ADMIN_PASSWORD - ); - if (!valid) { - console.log( - '\n', - boxen(`${error}${hint ? `\n\n${hint}` : ''}`, { - title: 'Invalid admin password', - padding: 1, - borderStyle: 'double', - titleAlignment: 'center', - borderColor: 'red', - }), - '\n' - ); - process.exit(1); - } - - let ncMeta; - try { - ncMeta = await _ncMeta.startTransaction(); - const email = process.env.NC_ADMIN_EMAIL.toLowerCase().trim(); - - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - salt - ); - const email_verification_token = uuidv4(); - const roles = 'user,super'; - - // if super admin not present - if (await User.isFirst(ncMeta)) { - // roles = 'owner,creator,editor' - T.emit('evt', { - evt_type: 'project:invite', - count: 1, - }); - - await User.insert( - { - firstname: '', - lastname: '', - email, - salt, - password, - email_verification_token, - roles, - }, - ncMeta - ); - } else { - const salt = await promisify(bcrypt.genSalt)(10); - const password = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - salt - ); - const email_verification_token = uuidv4(); - const superUser = await ncMeta.metaGet2(null, null, MetaTable.USERS, { - roles: 'user,super', - }); - - if (!superUser?.id) { - const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); - if (existingUserWithNewEmail?.id) { - // clear cache - await NocoCache.delAll( - CacheScope.USER, - `${existingUserWithNewEmail.email}___*` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.id}` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.email}` - ); - - // Update email and password of super admin account - await User.update( - existingUserWithNewEmail.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - roles, - }, - ncMeta - ); - } else { - T.emit('evt', { - evt_type: 'project:invite', - count: 1, - }); - - await User.insert( - { - firstname: '', - lastname: '', - email, - salt, - password, - email_verification_token, - roles, - }, - ncMeta - ); - } - } else if (email !== superUser.email) { - // update admin email and password and migrate projects - // if user already present and associated with some project - - // check user account already present with the new admin email - const existingUserWithNewEmail = await User.getByEmail(email, ncMeta); - - if (existingUserWithNewEmail?.id) { - // get all project access belongs to the existing account - // and migrate to the admin account - const existingUserProjects = await ncMeta.metaList2( - null, - null, - MetaTable.PROJECT_USERS, - { - condition: { fk_user_id: existingUserWithNewEmail.id }, - } - ); - - for (const existingUserProject of existingUserProjects) { - const userProject = await ProjectUser.get( - existingUserProject.project_id, - superUser.id, - ncMeta - ); - - // if admin user already have access to the project - // then update role based on the highest access level - if (userProject) { - if ( - rolesLevel[userProject.roles] > - rolesLevel[existingUserProject.roles] - ) { - await ProjectUser.update( - userProject.project_id, - superUser.id, - existingUserProject.roles, - ncMeta - ); - } - } else { - // if super doesn't have access then add the access - await ProjectUser.insert( - { - ...existingUserProject, - fk_user_id: superUser.id, - }, - ncMeta - ); - } - // delete the old project access entry from DB - await ProjectUser.delete( - existingUserProject.project_id, - existingUserProject.fk_user_id, - ncMeta - ); - } - - // delete existing user - await ncMeta.metaDelete( - null, - null, - MetaTable.USERS, - existingUserWithNewEmail.id - ); - - // clear cache - await NocoCache.delAll( - CacheScope.USER, - `${existingUserWithNewEmail.email}___*` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.id}` - ); - await NocoCache.del( - `${CacheScope.USER}:${existingUserWithNewEmail.email}` - ); - - // Update email and password of super admin account - await User.update( - superUser.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } else { - // if email's are not different update the password and hash - await User.update( - superUser.id, - { - salt, - email, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } - } else { - const newPasswordHash = await promisify(bcrypt.hash)( - process.env.NC_ADMIN_PASSWORD, - superUser.salt - ); - - if (newPasswordHash !== superUser.password) { - // if email's are same and passwords are different - // then update the password and token version - await User.update( - superUser.id, - { - salt, - password, - email_verification_token, - token_version: null, - refresh_token: null, - }, - ncMeta - ); - } - } - } - await ncMeta.commit(); - } catch (e) { - console.log('Error occurred while updating/creating admin user'); - console.log(e); - await ncMeta.rollback(e); - } - } -} diff --git a/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts b/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts deleted file mode 100644 index 85fc8bbf9c..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/initStrategies.ts +++ /dev/null @@ -1,336 +0,0 @@ -import { OrgUserRoles } from 'nocodb-sdk'; -import User from '../../models/User'; -import ProjectUser from '../../models/ProjectUser'; -import { promisify } from 'util'; -import { Strategy as CustomStrategy } from 'passport-custom'; -import passport from 'passport'; -import passportJWT from 'passport-jwt'; -import { Strategy as AuthTokenStrategy } from 'passport-auth-token'; -import { Strategy as GoogleStrategy } from 'passport-google-oauth20'; - -const PassportLocalStrategy = require('passport-local').Strategy; -const ExtractJwt = passportJWT.ExtractJwt; -const JwtStrategy = passportJWT.Strategy; - -const jwtOptions = { - jwtFromRequest: ExtractJwt.fromHeader('xc-auth'), -}; - -import bcrypt from 'bcryptjs'; -import Project from '../../models/Project'; -import NocoCache from '../../cache/NocoCache'; -import { CacheGetType, CacheScope } from '../../utils/globals'; -import ApiToken from '../../models/ApiToken'; -import Noco from '../../Noco'; -import Plugin from '../../models/Plugin'; -import { registerNewUserIfAllowed } from './userApis'; - -export function initStrategies(router): void { - passport.use( - 'authtoken', - new AuthTokenStrategy( - { headerFields: ['xc-token'], passReqToCallback: true }, - (req, token, done) => { - ApiToken.getByToken(token) - .then((apiToken) => { - if (!apiToken) { - return done({ msg: 'Invalid token' }); - } - - if (!apiToken.fk_user_id) return done(null, { roles: 'editor' }); - User.get(apiToken.fk_user_id) - .then((user) => { - user['is_api_token'] = true; - if (req.ncProjectId) { - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - // todo : cache - // await NocoCache.set(`${CacheScope.USER}:${key}`, user); - done(null, user); - }) - .catch((e) => done(e)); - } else { - return done(null, user); - } - }) - .catch((e) => { - console.log(e); - done({ msg: 'User not found' }); - }); - }) - .catch((e) => { - console.log(e); - done({ msg: 'Invalid token' }); - }); - } - ) - ); - - passport.serializeUser(function ( - { - id, - email, - email_verified, - roles: _roles, - provider, - firstname, - lastname, - isAuthorized, - isPublicBase, - token_version, - }, - done - ) { - const roles = (_roles || '') - .split(',') - .reduce((obj, role) => Object.assign(obj, { [role]: true }), {}); - if (roles.owner) { - roles.creator = true; - } - done(null, { - isAuthorized, - isPublicBase, - id, - email, - email_verified, - provider, - firstname, - lastname, - roles, - token_version, - }); - }); - - passport.deserializeUser(function (user, done) { - done(null, user); - }); - - passport.use( - new JwtStrategy( - { - secretOrKey: Noco.getConfig().auth.jwt.secret, - ...jwtOptions, - passReqToCallback: true, - ...Noco.getConfig().auth.jwt.options, - }, - async (req, jwtPayload, done) => { - // todo: improve this - if ( - req.ncProjectId && - jwtPayload.roles?.split(',').includes(OrgUserRoles.SUPER_ADMIN) - ) { - return User.getByEmail(jwtPayload?.email).then(async (user) => { - return done(null, { - ...user, - roles: `owner,creator,${OrgUserRoles.SUPER_ADMIN}`, - }); - }); - } - - const keyVals = [jwtPayload?.email]; - if (req.ncProjectId) { - keyVals.push(req.ncProjectId); - } - const key = keyVals.join('___'); - const cachedVal = await NocoCache.get( - `${CacheScope.USER}:${key}`, - CacheGetType.TYPE_OBJECT - ); - - if (cachedVal) { - if ( - !cachedVal.token_version || - !jwtPayload.token_version || - cachedVal.token_version !== jwtPayload.token_version - ) { - return done(new Error('Token Expired. Please login again.')); - } - return done(null, cachedVal); - } - - User.getByEmail(jwtPayload?.email) - .then(async (user) => { - if ( - !user.token_version || - !jwtPayload.token_version || - user.token_version !== jwtPayload.token_version - ) { - return done(new Error('Token Expired. Please login again.')); - } - if (req.ncProjectId) { - // this.xcMeta - // .metaGet(req.ncProjectId, null, 'nc_projects_users', { - // user_id: user?.id - // }) - - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - - await NocoCache.set(`${CacheScope.USER}:${key}`, user); - done(null, user); - }) - .catch((e) => done(e)); - } else { - // const roles = projectUser?.roles ? JSON.parse(projectUser.roles) : {guest: true}; - if (user) { - await NocoCache.set(`${CacheScope.USER}:${key}`, user); - return done(null, user); - } else { - return done(new Error('User not found')); - } - } - }) - .catch((err) => { - return done(err); - }); - } - ) - ); - - passport.use( - new PassportLocalStrategy( - { - usernameField: 'email', - session: false, - }, - async (email, password, done) => { - try { - const user = await User.getByEmail(email); - if (!user) { - return done({ msg: `Email ${email} is not registered!` }); - } - - if (!user.salt) { - return done({ - msg: `Please sign up with the invite token first or reset the password by clicking Forgot your password.`, - }); - } - - const hashedPassword = await promisify(bcrypt.hash)( - password, - user.salt - ); - if (user.password !== hashedPassword) { - return done({ msg: `Password not valid!` }); - } else { - return done(null, user); - } - } catch (e) { - done(e); - } - } - ) - ); - - passport.use( - 'baseView', - new CustomStrategy(async (req: any, callback) => { - let user; - if (req.headers['xc-shared-base-id']) { - // const cacheKey = `nc_shared_bases||${req.headers['xc-shared-base-id']}`; - - let sharedProject = null; - - if (!sharedProject) { - sharedProject = await Project.getByUuid( - req.headers['xc-shared-base-id'] - ); - } - user = { - roles: sharedProject?.roles, - }; - } - - callback(null, user); - }) - ); - - // mostly copied from older code - Plugin.getPluginByTitle('Google').then((googlePlugin) => { - if (googlePlugin && googlePlugin.input) { - const settings = JSON.parse(googlePlugin.input); - process.env.NC_GOOGLE_CLIENT_ID = settings.client_id; - process.env.NC_GOOGLE_CLIENT_SECRET = settings.client_secret; - } - - if ( - process.env.NC_GOOGLE_CLIENT_ID && - process.env.NC_GOOGLE_CLIENT_SECRET - ) { - const googleAuthParamsOrig = GoogleStrategy.prototype.authorizationParams; - GoogleStrategy.prototype.authorizationParams = (options: any) => { - const params = googleAuthParamsOrig.call(this, options); - - if (options.state) { - params.state = options.state; - } - - return params; - }; - - const clientConfig = { - clientID: process.env.NC_GOOGLE_CLIENT_ID, - clientSecret: process.env.NC_GOOGLE_CLIENT_SECRET, - // todo: update url - callbackURL: 'http://localhost:3000', - passReqToCallback: true, - }; - - const googleStrategy = new GoogleStrategy( - clientConfig, - async (req, _accessToken, _refreshToken, profile, done) => { - const email = profile.emails[0].value; - - User.getByEmail(email) - .then(async (user) => { - if (user) { - // if project id defined extract project level roles - if (req.ncProjectId) { - ProjectUser.get(req.ncProjectId, user.id) - .then(async (projectUser) => { - user.roles = projectUser?.roles || user.roles; - user.roles = - user.roles === 'owner' ? 'owner,creator' : user.roles; - // + (user.roles ? `,${user.roles}` : ''); - - done(null, user); - }) - .catch((e) => done(e)); - } else { - return done(null, user); - } - // if user not found create new user if allowed - // or return error - } else { - const salt = await promisify(bcrypt.genSalt)(10); - const user = await registerNewUserIfAllowed({ - firstname: null, - lastname: null, - email_verification_token: null, - email: profile.emails[0].value, - password: '', - salt, - }); - return done(null, user); - } - }) - .catch((err) => { - return done(err); - }); - } - ); - - passport.use(googleStrategy); - } - }); - - router.use(passport.initialize()); -} diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts deleted file mode 100644 index f7412b12ba..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/auth/emailVerify.ts +++ /dev/null @@ -1,70 +0,0 @@ -export default ` - - - NocoDB - Verify Email - - - - - - - -
- - - - - - Email verified successfully! - - - {{errMsg}} - - - - - - - -
- - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts deleted file mode 100644 index 514fc6d739..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/auth/resetPassword.ts +++ /dev/null @@ -1,108 +0,0 @@ -export default ` - - - NocoDB - Reset Password - - - - - - - -
- - - - - - Password reset successful! - - - - - - -
- - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts deleted file mode 100644 index afb2f5849a..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/forgotPassword.ts +++ /dev/null @@ -1,171 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts deleted file mode 100644 index fc81f9409e..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/invite.ts +++ /dev/null @@ -1,208 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts b/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts deleted file mode 100644 index 11702cc659..0000000000 --- a/packages/nocodb/src/lib/controllers/userApi/ui/emailTemplates/verify.ts +++ /dev/null @@ -1,207 +0,0 @@ -export default ` - - - - - Simple Transactional Email - - - - - - - - - - - - - -`; diff --git a/packages/nocodb/src/lib/meta/api/index.ts b/packages/nocodb/src/lib/meta/api/index.ts index 06da16fd10..f1a676997a 100644 --- a/packages/nocodb/src/lib/meta/api/index.ts +++ b/packages/nocodb/src/lib/meta/api/index.ts @@ -22,12 +22,12 @@ import hookApis from '../../controllers/hookController'; import pluginApis from '../../controllers/pluginController'; import gridViewColumnApis from '../../controllers/gridViewColumnController'; import kanbanViewApis from '../../controllers/kanbanViewController'; -import { userApis } from '../../controllers/userApi'; +import { userController } from '../../controllers/userController'; // import extractProjectIdAndAuthenticate from './helpers/extractProjectIdAndAuthenticate'; import utilApis from '../../controllers/utilController'; import projectUserApis from '../../controllers/projectUserController'; import sharedBaseApis from '../../controllers/sharedBaseController'; -import { initStrategies } from '../../controllers/userApi/initStrategies'; +import { initStrategies } from '../../controllers/userController/initStrategies'; import modelVisibilityApis from '../../controllers/modelVisibilityController'; import metaDiffApis from '../../controllers/metaDiffController'; import cacheApis from '../../controllers/cacheController'; @@ -51,9 +51,9 @@ import { Server, Socket } from 'socket.io'; import passport from 'passport'; import crypto from 'crypto'; -import swaggerApis from '../../controllers/swagger/swaggerApis'; -import importApis from '../../controllers/sync/importApis'; -import syncSourceApis from '../../controllers/sync/syncSourceApis'; +import swaggerApis from '../../controllers/swaggerController'; +import importApis from '../../controllers/syncController/importApis'; +import syncSourceApis from '../../controllers/syncController'; import mapViewApis from '../../controllers/mapViewController'; const clients: { [id: string]: Socket } = {}; @@ -108,7 +108,7 @@ export default function (router: Router, server) { router.use(kanbanViewApis); router.use(mapViewApis); - userApis(router); + userController(router); const io = new Server(server, { cors: { diff --git a/packages/nocodb/src/lib/services/attachmentService.ts b/packages/nocodb/src/lib/services/attachmentService.ts index f4a9bfdca1..1d1fc25f02 100644 --- a/packages/nocodb/src/lib/services/attachmentService.ts +++ b/packages/nocodb/src/lib/services/attachmentService.ts @@ -1,3 +1,6 @@ + + + // @ts-ignore import { Request, Response, Router } from 'express'; import { nanoid } from 'nanoid'; diff --git a/packages/nocodb/src/lib/services/projectUserService.ts b/packages/nocodb/src/lib/services/projectUserService.ts index 567568f488..2ff67913d4 100644 --- a/packages/nocodb/src/lib/services/projectUserService.ts +++ b/packages/nocodb/src/lib/services/projectUserService.ts @@ -282,7 +282,7 @@ export async function sendInviteEmail( ): Promise { try { const template = ( - await import('../meta/api/userApi/ui/emailTemplates/invite') + await import('./userService/ui/emailTemplates/invite') ).default; const emailAdapter = await NcPluginMgrv2.emailAdapter(); diff --git a/packages/nocodb/src/lib/services/tableService.ts b/packages/nocodb/src/lib/services/tableService.ts index 883927583f..1ff534548f 100644 --- a/packages/nocodb/src/lib/services/tableService.ts +++ b/packages/nocodb/src/lib/services/tableService.ts @@ -1,4 +1,3 @@ -// @ts-ignore import DOMPurify from 'isomorphic-dompurify'; import { AuditOperationSubTypes, @@ -28,6 +27,112 @@ import View from '../models/View'; import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; import { T } from 'nc-help'; +export async function tableUpdate(param: { + tableId: any; + table: TableReqType & { project_id?: string }; + projectId?: string; +}) { + const model = await Model.get(param.tableId); + + const project = await Project.getWithInfo( + param.table.project_id || param.projectId + ); + const base = project.bases.find((b) => b.id === model.base_id); + + if (model.project_id !== project.id) { + NcError.badRequest('Model does not belong to project'); + } + + // if meta present update meta and return + // todo: allow user to update meta and other prop in single api call + if ('meta' in param.table) { + await Model.updateMeta(param.tableId, param.table.meta); + + return { msg: 'success' } + } + + if (!param.table.table_name) { + NcError.badRequest( + 'Missing table name `table_name` property in request body' + ); + } + + if (base.is_meta && project.prefix) { + if (!param.table.table_name.startsWith(project.prefix)) { + param.table.table_name = `${project.prefix}${param.table.table_name}`; + } + } + + param.table.table_name = DOMPurify.sanitize(param.table.table_name); + + // validate table name + if (/^\s+|\s+$/.test(param.table.table_name)) { + NcError.badRequest( + 'Leading or trailing whitespace not allowed in table names' + ); + } + + if ( + !(await Model.checkTitleAvailable({ + table_name: param.table.table_name, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table name'); + } + + if (!param.table.title) { + param.table.title = getTableNameAlias( + param.table.table_name, + project.prefix, + base + ); + } + + if ( + !(await Model.checkAliasAvailable({ + title: param.table.title, + project_id: project.id, + base_id: base.id, + })) + ) { + NcError.badRequest('Duplicate table alias'); + } + + const sqlMgr = await ProjectMgrv2.getSqlMgr(project); + const sqlClient = await NcConnectionMgrv2.getSqlClient(base); + + let tableNameLengthLimit = 255; + const sqlClientType = sqlClient.knex.clientType(); + if (sqlClientType === 'mysql2' || sqlClientType === 'mysql') { + tableNameLengthLimit = 64; + } else if (sqlClientType === 'pg') { + tableNameLengthLimit = 63; + } else if (sqlClientType === 'mssql') { + tableNameLengthLimit = 128; + } + + if (param.table.table_name.length > tableNameLengthLimit) { + NcError.badRequest(`Table name exceeds ${tableNameLengthLimit} characters`); + } + + await Model.updateAliasAndTableName( + param.tableId, + param.table.title, + param.table.table_name + ); + + await sqlMgr.sqlOpPlus(base, 'tableRename', { + ...param.table, + tn: param.table.table_name, + tn_old: model.table_name, + }); + + T.emit('evt', { evt_type: 'table:updated' }); + return true; +} + export function reorderTable(param: { tableId: string; order: any }) { return Model.updateOrder(param.tableId, param.order); } @@ -100,7 +205,7 @@ export async function getTableWithAccessibleViews(param: { // todo: optimise const viewList = await xcVisibilityMetaGet(table.project_id, [table]); - //await View.list(req.params.tableId) + //await View.list(param.tableId) table.views = viewList.filter((table: any) => { return Object.keys(param.user?.roles).some( (role) => param.user?.roles[role] && !table.disabled[role] diff --git a/packages/nocodb/src/lib/services/userService/helpers.ts b/packages/nocodb/src/lib/services/userService/helpers.ts index 90e2d81dfc..387b51170e 100644 --- a/packages/nocodb/src/lib/services/userService/helpers.ts +++ b/packages/nocodb/src/lib/services/userService/helpers.ts @@ -1,7 +1,7 @@ import * as jwt from 'jsonwebtoken'; import crypto from 'crypto'; +import User from '../../models/User'; import { NcConfig } from '../../../interface/config'; -import { User } from '../../models'; export function genJwt(user: User, config: NcConfig) { return jwt.sign(