diff --git a/packages/nocodb-nest/src/app.module.ts b/packages/nocodb-nest/src/app.module.ts index 08a2fb7ff5..6ee483ff45 100644 --- a/packages/nocodb-nest/src/app.module.ts +++ b/packages/nocodb-nest/src/app.module.ts @@ -51,6 +51,7 @@ import type { MiddlewareConsumer, OnApplicationBootstrap, } from '@nestjs/common'; +import NcUpgrader from './version-upgrader/NcUpgrader'; @Module({ imports: [ @@ -130,5 +131,8 @@ export class AppModule implements OnApplicationBootstrap { // temporary hack Noco._ncMeta = this.metaService; Noco.config = this.connection.config; + + // run upgrader + await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta }); } } diff --git a/packages/nocodb-nest/src/interface/XcDynamicChanges.ts b/packages/nocodb-nest/src/interface/XcDynamicChanges.ts deleted file mode 100644 index 15248ed68a..0000000000 --- a/packages/nocodb-nest/src/interface/XcDynamicChanges.ts +++ /dev/null @@ -1,8 +0,0 @@ -export default interface XcDynamicChanges { - onTableCreate(tn: string): Promise; - onTableUpdate(changeObj: any): Promise; - onTableDelete(tn: string): Promise; - onTableRename(oldTableName: string, newTableName: string): Promise; - onHandlerCodeUpdate(tn: string): Promise; - onMetaUpdate(tn: string): Promise; -} diff --git a/packages/nocodb-nest/src/version-upgrader/NcUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/NcUpgrader.ts new file mode 100644 index 0000000000..c217294c6e --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/NcUpgrader.ts @@ -0,0 +1,151 @@ +import debug from 'debug'; +import { T } from 'nc-help'; +import boxen from 'boxen'; +import { NcConfig } from '../interface/config'; +import { MetaService } from '../meta/meta.service'; +import ncAttachmentUpgrader from './ncAttachmentUpgrader'; +import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002'; +import ncStickyColumnUpgrader from './ncStickyColumnUpgrader'; +import ncFilterUpgrader_0104004 from './ncFilterUpgrader_0104004'; +import ncFilterUpgrader_0105003 from './ncFilterUpgrader_0105003'; +import ncFilterUpgrader from './ncFilterUpgrader'; +import ncProjectRolesUpgrader from './ncProjectRolesUpgrader'; +import ncDataTypesUpgrader from './ncDataTypesUpgrader'; +import ncProjectUpgraderV2_0090000 from './ncProjectUpgraderV2_0090000'; +import ncProjectEnvUpgrader0011045 from './ncProjectEnvUpgrader0011045'; +import ncProjectEnvUpgrader from './ncProjectEnvUpgrader'; +import ncHookUpgrader from './ncHookUpgrader'; + +const log = debug('nc:version-upgrader'); + +export interface NcUpgraderCtx { + ncMeta: MetaService; +} + +export default class NcUpgrader { + private static STORE_KEY = 'NC_CONFIG_MAIN'; + + // Todo: transaction + public static async upgrade(ctx: NcUpgraderCtx): Promise { + this.log(`upgrade :`); + let oldVersion; + + try { + ctx.ncMeta = await ctx.ncMeta.startTransaction(); + + const NC_VERSIONS: any[] = [ + { name: '0009000', handler: null }, + { name: '0009044', handler: null }, + { name: '0011043', handler: ncProjectEnvUpgrader }, + { name: '0011045', handler: ncProjectEnvUpgrader0011045 }, + { name: '0090000', handler: ncProjectUpgraderV2_0090000 }, + { name: '0098004', handler: ncDataTypesUpgrader }, + { name: '0098005', handler: ncProjectRolesUpgrader }, + { name: '0100002', handler: ncFilterUpgrader }, + { name: '0101002', handler: ncAttachmentUpgrader }, + { name: '0104002', handler: ncAttachmentUpgrader_0104002 }, + { name: '0104004', handler: ncFilterUpgrader_0104004 }, + { name: '0105002', handler: ncStickyColumnUpgrader }, + { name: '0105003', handler: ncFilterUpgrader_0105003 }, + { name: '0105004', handler: ncHookUpgrader }, + ]; + if (!(await ctx.ncMeta.knexConnection?.schema?.hasTable?.('nc_store'))) { + return; + } + this.log(`upgrade : Getting configuration from meta database`); + + const config = await ctx.ncMeta.metaGet('', '', 'nc_store', { + key: this.STORE_KEY, + }); + + if (config) { + const configObj: NcConfig = JSON.parse(config.value); + if (configObj.version !== process.env.NC_VERSION) { + oldVersion = configObj.version; + for (const version of NC_VERSIONS) { + // compare current version and old version + if (version.name > configObj.version) { + this.log( + `upgrade : Upgrading '%s' => '%s'`, + configObj.version, + version.name, + ); + await version?.handler?.(ctx); + + // update version in meta after each upgrade + config.version = version.name; + await ctx.ncMeta.metaUpdate( + '', + '', + 'nc_store', + { + value: JSON.stringify(config), + }, + { + key: NcUpgrader.STORE_KEY, + }, + ); + + // todo: backup data + } + if (version.name === process.env.NC_VERSION) { + break; + } + } + config.version = process.env.NC_VERSION; + } + } else { + this.log(`upgrade : Inserting config to meta database`); + const configObj: any = {}; + const isOld = (await ctx.ncMeta.projectList())?.length; + configObj.version = isOld ? '0009000' : process.env.NC_VERSION; + await ctx.ncMeta.metaInsert('', '', 'nc_store', { + key: NcUpgrader.STORE_KEY, + value: JSON.stringify(configObj), + }); + if (isOld) { + await this.upgrade(ctx); + } + } + await ctx.ncMeta.commit(); + T.emit('evt', { + evt_type: 'appMigration:upgraded', + from: oldVersion, + to: process.env.NC_VERSION, + }); + } catch (e) { + await ctx.ncMeta.rollback(e); + T.emit('evt', { + evt_type: 'appMigration:failed', + from: oldVersion, + to: process.env.NC_VERSION, + msg: e.message, + err: e?.stack?.split?.('\n').slice(0, 2).join('\n'), + }); + console.log(getUpgradeErrorLog(e, oldVersion, process.env.NC_VERSION)); + throw e; + } + } + + private static log(str, ...args): void { + log(`${str}`, ...args); + } +} + +function getUpgradeErrorLog(e: Error, oldVersion: string, newVersion: string) { + const errorTitle = `Migration from ${oldVersion} to ${newVersion} failed`; + + return boxen( + `Error +----- +${e.stack} + + +Please raise an issue in our github by using following link : +https://github.com/nocodb/nocodb/issues/new?labels=Type%3A%20Bug&template=bug_report.md + +Or contact us in our Discord community by following link : +https://discord.gg/5RgZmkW ( message @o1lab, @pranavxc or @wingkwong )`, + { title: errorTitle, padding: 1, borderColor: 'yellow' }, + ); +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader.ts new file mode 100644 index 0000000000..cc318e644f --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader.ts @@ -0,0 +1,190 @@ +import { UITypes } from 'nocodb-sdk'; +import { XKnex } from '../db/CustomKnex'; +import { MetaTable } from '../utils/globals'; +import Base from '../models/Base'; +import Model from '../models/Model'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { throwTimeoutError } from './ncUpgradeErrors'; +import type { Knex } from 'knex'; +import type { NcUpgraderCtx } from './NcUpgrader'; +// import type { XKnex } from '../db/sql-data-mapper'; +import type { BaseType } from 'nocodb-sdk'; + +// before 0.103.0, an attachment object was like +// [{ +// "url": "http://localhost:8080/download/noco/xcdb/Sheet-1/title5/39A410.jpeg", +// "title": "foo.jpeg", +// "mimetype": "image/jpeg", +// "size": 6494 +// }] +// in this way, if the base url is changed, the url will be broken +// this upgrader is to convert the existing local attachment object to the following format +// [{ +// "url": "http://localhost:8080/download/noco/xcdb/Sheet-1/title5/39A410.jpeg", +// "path": "download/noco/xcdb/Sheet-1/title5/39A410.jpeg", +// "title": "foo.jpeg", +// "mimetype": "image/jpeg", +// "size": 6494 +// }] +// the new url will be constructed by `${ncSiteUrl}/${path}` in UI. the old url will be used for fallback +// while other non-local attachments will remain unchanged + +function getTnPath(knex: XKnex, tb: Model) { + const schema = (knex as any).searchPath?.(); + const clientType = knex.clientType(); + if (clientType === 'mssql' && schema) { + return knex.raw('??.??', [schema, tb.table_name]).toQuery(); + } else if (clientType === 'snowflake') { + return [ + knex.client.config.connection.database, + knex.client.config.connection.schema, + tb.table_name, + ].join('.'); + } else { + return tb.table_name; + } +} + +export default async function ({ ncMeta }: NcUpgraderCtx) { + const bases: BaseType[] = await ncMeta.metaList2(null, null, MetaTable.BASES); + for (const _base of bases) { + const base = new Base(_base); + + // skip if the project_id is missing + if (!base.project_id) { + continue; + } + + const project = await ncMeta.metaGet2(null, null, MetaTable.PROJECT, { + id: base.project_id, + }); + + // skip if the project is missing + if (!project) { + continue; + } + + const isProjectDeleted = project.deleted; + + const knex: Knex = base.is_meta + ? ncMeta.knexConnection + : await NcConnectionMgrv2.get(base); + const models = await base.getModels(ncMeta); + + // used in timeout error message + const timeoutErrorInfo = { + projectTitle: project.title, + connection: knex.client.config.connection, + }; + + for (const model of models) { + try { + // if the table is missing in database, skip + if (!(await knex.schema.hasTable(getTnPath(knex, model)))) { + continue; + } + + const updateRecords = []; + + // get all attachment & primary key columns + // and filter out the columns that are missing in database + const columns = await (await Model.get(model.id, ncMeta)) + .getColumns(ncMeta) + .then(async (columns) => { + const filteredColumns = []; + + for (const column of columns) { + if (column.uidt !== UITypes.Attachment && !column.pk) continue; + if ( + !(await knex.schema.hasColumn( + getTnPath(knex, model), + column.column_name, + )) + ) + continue; + filteredColumns.push(column); + } + + return filteredColumns; + }); + + const attachmentColumns = columns + .filter((c) => c.uidt === UITypes.Attachment) + .map((c) => c.column_name); + if (attachmentColumns.length === 0) { + continue; + } + const primaryKeys = columns + .filter((c) => c.pk) + .map((c) => c.column_name); + + const records = await knex(getTnPath(knex, model)).select(); + + for (const record of records) { + for (const attachmentColumn of attachmentColumns) { + let attachmentMeta: Array<{ + url: string; + }>; + + // if parsing failed ignore the cell + try { + attachmentMeta = + typeof record[attachmentColumn] === 'string' + ? JSON.parse(record[attachmentColumn]) + : record[attachmentColumn]; + } catch {} + + // if cell data is not an array, ignore it + if (!Array.isArray(attachmentMeta)) { + continue; + } + + if (attachmentMeta) { + const newAttachmentMeta = []; + for (const attachment of attachmentMeta) { + if ('url' in attachment && typeof attachment.url === 'string') { + const match = attachment.url.match(/^(.*)\/download\/(.*)$/); + if (match) { + // e.g. http://localhost:8080/download/noco/xcdb/Sheet-1/title5/ee2G8p_nute_gunray.png + // match[1] = http://localhost:8080 + // match[2] = download/noco/xcdb/Sheet-1/title5/ee2G8p_nute_gunray.png + const path = `download/${match[2]}`; + + newAttachmentMeta.push({ + ...attachment, + path, + }); + } else { + // keep it as it is + newAttachmentMeta.push(attachment); + } + } + } + const where = primaryKeys + .map((key) => { + return { [key]: record[key] }; + }) + .reduce((acc, val) => Object.assign(acc, val), {}); + updateRecords.push( + await knex(getTnPath(knex, model)) + .update({ + [attachmentColumn]: JSON.stringify(newAttachmentMeta), + }) + .where(where), + ); + } + } + } + await Promise.all(updateRecords); + } catch (e) { + // ignore the error related to deleted project + if (!isProjectDeleted) { + // throw the custom timeout error message if applicable + throwTimeoutError(e, timeoutErrorInfo); + // throw general error + throw e; + } + } + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader_0104002.ts b/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader_0104002.ts new file mode 100644 index 0000000000..d43ccfc861 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader_0104002.ts @@ -0,0 +1,170 @@ +import { UITypes } from 'nocodb-sdk'; +import { XKnex } from '../db/CustomKnex' +import { MetaTable } from '../utils/globals'; +import Base from '../models/Base'; +import Model from '../models/Model'; +import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2'; +import { throwTimeoutError } from './ncUpgradeErrors'; +import type { Knex } from 'knex'; +import type { NcUpgraderCtx } from './NcUpgrader'; +import type { BaseType } from 'nocodb-sdk'; + +// after 0101002 upgrader, the attachment object would become broken when +// (1) switching views after updating a singleSelect field +// since `url` will be enriched the attachment cell, and `saveOrUpdateRecords` in Grid.vue will be triggered +// in this way, the attachment value will be corrupted like +// {"{\"path\":\"download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg\",\"title\":\"haha.jpeg\",\"mimetype\":\"image/jpeg\",\"size\":6494,\"url\":\"http://localhost:8080/download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg\"}"} +// while the expected one is +// [{"path":"download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg","title":"haha.jpeg","mimetype":"image/jpeg","size":6494}] +// (2) or reordering attachments +// since the incoming value is not string, the value will be broken +// hence, this upgrader is to revert back these corrupted values + +function getTnPath(knex: XKnex, tb: Model) { + const schema = (knex as any).searchPath?.(); + const clientType = knex.clientType(); + if (clientType === 'mssql' && schema) { + return knex.raw('??.??', [schema, tb.table_name]).toQuery(); + } else if (clientType === 'snowflake') { + return [ + knex.client.config.connection.database, + knex.client.config.connection.schema, + tb.table_name, + ].join('.'); + } else { + return tb.table_name; + } +} + +export default async function ({ ncMeta }: NcUpgraderCtx) { + const bases: BaseType[] = await ncMeta.metaList2(null, null, MetaTable.BASES); + for (const _base of bases) { + const base = new Base(_base); + + // skip if the project_id is missing + if (!base.project_id) { + continue; + } + + const project = await ncMeta.metaGet2(null, null, MetaTable.PROJECT, { + id: base.project_id, + }); + + // skip if the project is missing + if (!project) { + continue; + } + + const isProjectDeleted = project.deleted; + + const knex: Knex = base.is_meta + ? ncMeta.knexConnection + : await NcConnectionMgrv2.get(base); + const models = await base.getModels(ncMeta); + + // used in timeout error message + const timeoutErrorInfo = { + projectTitle: project.title, + connection: knex.client.config.connection, + }; + + for (const model of models) { + try { + // if the table is missing in database, skip + if (!(await knex.schema.hasTable(getTnPath(knex, model)))) { + continue; + } + + const updateRecords = []; + + // get all attachment & primary key columns + // and filter out the columns that are missing in database + const columns = await (await Model.get(model.id, ncMeta)) + .getColumns(ncMeta) + .then(async (columns) => { + const filteredColumns = []; + + for (const column of columns) { + if (column.uidt !== UITypes.Attachment && !column.pk) continue; + if ( + !(await knex.schema.hasColumn( + getTnPath(knex, model), + column.column_name + )) + ) + continue; + filteredColumns.push(column); + } + + return filteredColumns; + }); + + const attachmentColumns = columns + .filter((c) => c.uidt === UITypes.Attachment) + .map((c) => c.column_name); + if (attachmentColumns.length === 0) { + continue; + } + const primaryKeys = columns + .filter((c) => c.pk) + .map((c) => c.column_name); + + const records = await knex(getTnPath(knex, model)).select(); + + for (const record of records) { + const where = primaryKeys + .map((key) => { + return { [key]: record[key] }; + }) + .reduce((acc, val) => Object.assign(acc, val), {}); + for (const attachmentColumn of attachmentColumns) { + if (typeof record[attachmentColumn] === 'string') { + // potentially corrupted + try { + JSON.parse(record[attachmentColumn]); + // it works fine - skip + continue; + } catch { + try { + // corrupted + let corruptedAttachment = record[attachmentColumn]; + // replace the first and last character with `[` and `]` + // and parse it again + corruptedAttachment = JSON.parse( + `[${corruptedAttachment.slice( + 1, + corruptedAttachment.length - 1 + )}]` + ); + const newAttachmentMeta = []; + for (const attachment of corruptedAttachment) { + newAttachmentMeta.push(JSON.parse(attachment)); + } + updateRecords.push( + await knex(getTnPath(knex, model)) + .update({ + [attachmentColumn]: JSON.stringify(newAttachmentMeta), + }) + .where(where) + ); + } catch { + // if parsing failed ignore the cell + continue; + } + } + } + } + } + await Promise.all(updateRecords); + } catch (e) { + // ignore the error related to deleted project + if (!isProjectDeleted) { + // throw the custom timeout error message if applicable + throwTimeoutError(e, timeoutErrorInfo); + // throw general error + throw e; + } + } + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncDataTypesUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncDataTypesUpgrader.ts new file mode 100644 index 0000000000..53bc82a0ea --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncDataTypesUpgrader.ts @@ -0,0 +1,14 @@ +import { UITypes } from 'nocodb-sdk'; +import { MetaTable } from '../utils/globals'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +// The Count and AutoNumber types are removed +// so convert all existing Count and AutoNumber fields to Number type +export default async function (ctx: NcUpgraderCtx) { + // directly update uidt of all existing Count and AutoNumber fields to Number + await ctx.ncMeta.knex + .update({ uidt: UITypes.Number }) + .where({ uidt: UITypes.Count }) + .orWhere({ uidt: UITypes.AutoNumber }) + .table(MetaTable.COLUMNS); +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader.ts new file mode 100644 index 0000000000..bd5cefd60f --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader.ts @@ -0,0 +1,40 @@ +import { MetaTable } from '../utils/globals'; +import View from '../models/View'; +import Hook from '../models/Hook'; +import Column from '../models/Column'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +// before 0.101.0, an incorrect project_id was inserted when +// a filter is created without specifying the column +// this upgrader is to retrieve the correct project id from either view, hook, or column +// and update the project id +export default async function ({ ncMeta }: NcUpgraderCtx) { + const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); + for (const filter of filters) { + let model: { project_id?: string; base_id?: string }; + if (filter.fk_view_id) { + model = await View.get(filter.fk_view_id, ncMeta); + } else if (filter.fk_hook_id) { + model = await Hook.get(filter.fk_hook_id, ncMeta); + } else if (filter.fk_column_id) { + model = await Column.get({ colId: filter.fk_column_id }, ncMeta); + } else { + continue; + } + + // skip if related model is not found + if (!model) { + continue; + } + + if (filter.project_id !== model.project_id) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.FILTER_EXP, + { base_id: model.base_id, project_id: model.project_id }, + filter.id + ); + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0104004.ts b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0104004.ts new file mode 100644 index 0000000000..477be1f64b --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0104004.ts @@ -0,0 +1,359 @@ +import { UITypes } from 'nocodb-sdk'; +import { MetaService } from '../meta/meta.service' +import { MetaTable } from '../utils/globals'; +import Column from '../models/Column'; +import Filter from '../models/Filter'; +import Project from '../models/Project'; +import type { NcUpgraderCtx } from './NcUpgrader'; +import type { SelectOptionsType } from 'nocodb-sdk'; + +// as of 0.104.3, almost all filter operators are available to all column types +// while some of them aren't supposed to be shown +// this upgrader is to remove those unsupported filters / migrate to the correct filter + +// Change Summary: +// - Text-based columns: +// - remove `>`, `<`, `>=`, `<=` +// - Numeric-based / SingleSelect columns: +// - remove `like` +// - migrate `null`, and `empty` to `blank` +// - Checkbox columns: +// - remove `equal` +// - migrate `empty` and `null` to `notchecked` +// - MultiSelect columns: +// - remove `like` +// - migrate `equal`, `null`, `empty` +// - Attachment columns: +// - remove `>`, `<`, `>=`, `<=`, `equal` +// - migrate `empty`, `null` to `blank` +// - LTAR columns: +// - remove `>`, `<`, `>=`, `<=` +// - migrate `empty`, `null` to `blank` +// - Lookup columns: +// - migrate `empty`, `null` to `blank` +// - Duration columns: +// - remove `like` +// - migrate `empty`, `null` to `blank` + +const removeEqualFilters = (filter, ncMeta) => { + const actions = []; + // remove `is equal`, `is not equal` + if (['eq', 'neq'].includes(filter.comparison_op)) { + actions.push(Filter.delete(filter.id, ncMeta)); + } + return actions; +}; + +const removeArithmeticFilters = (filter, ncMeta) => { + const actions = []; + // remove `>`, `<`, `>=`, `<=` + if (['gt', 'lt', 'gte', 'lte'].includes(filter.comparison_op)) { + actions.push(Filter.delete(filter.id, ncMeta)); + } + return actions; +}; + +const removeLikeFilters = (filter, ncMeta) => { + const actions = []; + // remove `is like`, `is not like` + if (['like', 'nlike'].includes(filter.comparison_op)) { + actions.push(Filter.delete(filter.id, ncMeta)); + } + return actions; +}; + +const migrateNullAndEmptyToBlankFilters = (filter, ncMeta) => { + const actions = []; + if (['empty', 'null'].includes(filter.comparison_op)) { + // migrate to blank + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'blank', + }, + ncMeta + ) + ); + } else if (['notempty', 'notnull'].includes(filter.comparison_op)) { + // migrate to not blank + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'notblank', + }, + ncMeta + ) + ); + } + return actions; +}; + +const migrateMultiSelectEq = async (filter, col: Column, ncMeta) => { + // only allow eq / neq + if (!['eq', 'neq'].includes(filter.comparison_op)) return; + // if there is no value -> delete this filter + if (!filter.value) { + return await Filter.delete(filter.id, ncMeta); + } + // options inputted from users + const options = filter.value.split(','); + // retrieve the possible col options + const colOptions = (await col.getColOptions()) as SelectOptionsType; + // only include valid options as the input value becomes dropdown type now + const validOptions = []; + for (const option of options) { + if (colOptions.options.includes(option)) { + validOptions.push(option); + } + } + const newFilterValue = validOptions.join(','); + // if all inputted options are invalid -> delete this filter + if (!newFilterValue) { + return await Filter.delete(filter.id, ncMeta); + } + const actions = []; + if (filter.comparison_op === 'eq') { + // migrate to `contains all of` + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'anyof', + value: newFilterValue, + }, + ncMeta + ) + ); + } else if (filter.comparison_op === 'neq') { + // migrate to `doesn't contain all of` + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'nanyof', + value: newFilterValue, + }, + ncMeta + ) + ); + } + return await Promise.all(actions); +}; + +const migrateToCheckboxFilter = (filter, ncMeta) => { + const actions = []; + const possibleTrueValues = ['true', 'True', '1', 'T', 'Y']; + const possibleFalseValues = ['false', 'False', '0', 'F', 'N']; + if (['empty', 'null'].includes(filter.comparison_op)) { + // migrate to not checked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'notchecked', + }, + ncMeta + ) + ); + } else if (['notempty', 'notnull'].includes(filter.comparison_op)) { + // migrate to checked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'checked', + }, + ncMeta + ) + ); + } else if (filter.comparison_op === 'eq') { + if (possibleTrueValues.includes(filter.value)) { + // migrate to checked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'checked', + value: '', + }, + ncMeta + ) + ); + } else if (possibleFalseValues.includes(filter.value)) { + // migrate to notchecked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'notchecked', + value: '', + }, + ncMeta + ) + ); + } else { + // invalid value - good to delete + actions.push(Filter.delete(filter.id, ncMeta)); + } + } else if (filter.comparison_op === 'neq') { + if (possibleFalseValues.includes(filter.value)) { + // migrate to checked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'checked', + value: '', + }, + ncMeta + ) + ); + } else if (possibleTrueValues.includes(filter.value)) { + // migrate to not checked + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'notchecked', + value: '', + }, + ncMeta + ) + ); + } else { + // invalid value - good to delete + actions.push(Filter.delete(filter.id, ncMeta)); + } + } + return actions; +}; + +async function migrateFilters(ncMeta: MetaService) { + const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); + for (const filter of filters) { + if (!filter.fk_column_id || filter.is_group) { + continue; + } + const col = await Column.get({ colId: filter.fk_column_id }, ncMeta); + if ( + [ + UITypes.SingleLineText, + UITypes.LongText, + UITypes.PhoneNumber, + UITypes.Email, + UITypes.URL, + ].includes(col.uidt) + ) { + await Promise.all(removeArithmeticFilters(filter, ncMeta)); + } else if ( + [ + // numeric fields + UITypes.Duration, + UITypes.Currency, + UITypes.Percent, + UITypes.Number, + UITypes.Decimal, + UITypes.Rating, + UITypes.Rollup, + // select fields + UITypes.SingleSelect, + ].includes(col.uidt) + ) { + await Promise.all([ + ...removeLikeFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + } else if (col.uidt === UITypes.Checkbox) { + await Promise.all(migrateToCheckboxFilter(filter, ncMeta)); + } else if (col.uidt === UITypes.MultiSelect) { + await Promise.all([ + ...removeLikeFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + await migrateMultiSelectEq(filter, col, ncMeta); + } else if (col.uidt === UITypes.Attachment) { + await Promise.all([ + ...removeArithmeticFilters(filter, ncMeta), + ...removeEqualFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + } else if (col.uidt === UITypes.LinkToAnotherRecord) { + await Promise.all([ + ...removeArithmeticFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + } else if (col.uidt === UITypes.Lookup) { + await Promise.all([ + ...removeArithmeticFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + } else if (col.uidt === UITypes.Duration) { + await Promise.all([ + ...removeLikeFilters(filter, ncMeta), + ...migrateNullAndEmptyToBlankFilters(filter, ncMeta), + ]); + } + } +} + +async function updateProjectMeta(ncMeta: MetaService) { + const projectHasEmptyOrFilters: Record = {}; + + const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); + + const actions = []; + + for (const filter of filters) { + if ( + ['notempty', 'notnull', 'empty', 'null'].includes(filter.comparison_op) + ) { + projectHasEmptyOrFilters[filter.project_id] = true; + } + } + + const projects = await ncMeta.metaList2(null, null, MetaTable.PROJECT); + + const defaultProjectMeta = { + showNullAndEmptyInFilter: false, + }; + + for (const project of projects) { + const oldProjectMeta = project.meta; + let newProjectMeta = defaultProjectMeta; + try { + newProjectMeta = + (typeof oldProjectMeta === 'string' + ? JSON.parse(oldProjectMeta) + : oldProjectMeta) ?? defaultProjectMeta; + } catch {} + + newProjectMeta = { + ...newProjectMeta, + showNullAndEmptyInFilter: projectHasEmptyOrFilters[project.id] ?? false, + }; + + actions.push( + Project.update( + project.id, + { + meta: JSON.stringify(newProjectMeta), + }, + ncMeta + ) + ); + } + await Promise.all(actions); +} + +export default async function ({ ncMeta }: NcUpgraderCtx) { + // fix the existing filter behaviours or + // migrate `null` or `empty` filters to `blank` + await migrateFilters(ncMeta); + // enrich `showNullAndEmptyInFilter` in project meta + // if there is empty / null filters in existing projects, + // then set `showNullAndEmptyInFilter` to true + // else set to false + await updateProjectMeta(ncMeta); +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0105003.ts b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0105003.ts new file mode 100644 index 0000000000..5c1fd559fe --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0105003.ts @@ -0,0 +1,101 @@ +import { UITypes } from 'nocodb-sdk'; +import { MetaService } from '../meta/meta.service'; +import { MetaTable } from '../utils/globals'; +import Column from '../models/Column'; +import Filter from '../models/Filter'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +// as of 0.105.3, year, time, date and datetime filters include `is like` and `is not like` which are not practical +// `removeLikeAndNlikeFilters` in this upgrader is simply to remove them + +// besides, `null` and `empty` will be migrated to `blank` in `migrateEmptyAndNullFilters` + +// since the upcoming version will introduce a set of new filters for date / datetime with a new `comparison_sub_op` +// `eq` and `neq` would become `is` / `is not` (comparison_op) + `exact date` (comparison_sub_op) +// `migrateEqAndNeqFilters` in this upgrader is to add `exact date` in comparison_sub_op + +// Change Summary: +// - Date / DateTime columns: +// - remove `is like` and `is not like` +// - migrate `null` or `empty` filters to `blank` +// - add `exact date` in comparison_sub_op for existing filters `eq` and `neq` +// - Year / Time columns: +// - remove `is like` and `is not like` +// - migrate `null` or `empty` filters to `blank` + +function removeLikeAndNlikeFilters(filter: Filter, ncMeta: MetaService) { + const actions = []; + // remove `is like` and `is not like` + if (['like', 'nlike'].includes(filter.comparison_op)) { + actions.push(Filter.delete(filter.id, ncMeta)); + } + return actions; +} + +function migrateEqAndNeqFilters(filter: Filter, ncMeta: MetaService) { + const actions = []; + // remove `is like` and `is not like` + if (['eq', 'neq'].includes(filter.comparison_op)) { + actions.push( + Filter.update( + filter.id, + { + comparison_sub_op: 'exactDate', + }, + ncMeta, + ), + ); + } + return actions; +} + +function migrateEmptyAndNullFilters(filter: Filter, ncMeta: MetaService) { + const actions = []; + // remove `is like` and `is not like` + if (['empty', 'null'].includes(filter.comparison_op)) { + // migrate to blank + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'blank', + }, + ncMeta, + ), + ); + } else if (['notempty', 'notnull'].includes(filter.comparison_op)) { + // migrate to not blank + actions.push( + Filter.update( + filter.id, + { + comparison_op: 'notblank', + }, + ncMeta, + ), + ); + } + return actions; +} + +export default async function ({ ncMeta }: NcUpgraderCtx) { + const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP); + for (const filter of filters) { + if (!filter.fk_column_id || filter.is_group) { + continue; + } + const col = await Column.get({ colId: filter.fk_column_id }, ncMeta); + if ([UITypes.Date, UITypes.DateTime].includes(col.uidt)) { + await Promise.all([ + ...removeLikeAndNlikeFilters(filter, ncMeta), + ...migrateEmptyAndNullFilters(filter, ncMeta), + ...migrateEqAndNeqFilters(filter, ncMeta), + ]); + } else if ([UITypes.Time, UITypes.Year].includes(col.uidt)) { + await Promise.all([ + ...removeLikeAndNlikeFilters(filter, ncMeta), + ...migrateEmptyAndNullFilters(filter, ncMeta), + ]); + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncHookUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncHookUpgrader.ts new file mode 100644 index 0000000000..661b44583d --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncHookUpgrader.ts @@ -0,0 +1,19 @@ +import { MetaTable } from '../utils/globals'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +export default async function ({ ncMeta }: NcUpgraderCtx) { + const actions = []; + const hooks = await ncMeta.metaList2(null, null, MetaTable.HOOKS); + for (const hook of hooks) { + actions.push( + ncMeta.metaUpdate( + null, + null, + MetaTable.HOOKS, + { version: 'v1' }, + hook.id, + ), + ); + } + await Promise.all(actions); +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader.ts new file mode 100644 index 0000000000..328365d68e --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader.ts @@ -0,0 +1,18 @@ +import type { NcUpgraderCtx } from './NcUpgrader'; + +export default async function (ctx: NcUpgraderCtx) { + const projects = await ctx.ncMeta.projectList(); + + for (const project of projects) { + const projectConfig = JSON.parse(project.config); + + const envVal = projectConfig.envs?.dev; + projectConfig.workingEnv = '_noco'; + + if (envVal) { + projectConfig.envs._noco = envVal; + delete projectConfig.envs.dev; + } + await ctx.ncMeta.projectUpdate(project?.id, projectConfig); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader0011045.ts b/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader0011045.ts new file mode 100644 index 0000000000..56fd587ca3 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader0011045.ts @@ -0,0 +1,11 @@ +import type { NcUpgraderCtx } from './NcUpgrader'; + +export default async function (ctx: NcUpgraderCtx) { + const projects = await ctx.ncMeta.projectList(); + + for (const project of projects) { + const projectConfig = JSON.parse(project.config); + projectConfig.env = '_noco'; + await ctx.ncMeta.projectUpdate(project?.id, projectConfig); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncProjectRolesUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncProjectRolesUpgrader.ts new file mode 100644 index 0000000000..f3e69083f8 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncProjectRolesUpgrader.ts @@ -0,0 +1,43 @@ +import { OrgUserRoles } from 'nocodb-sdk'; +import { NC_APP_SETTINGS } from '../constants'; +import Store from '../models/Store'; +import { MetaTable } from '../utils/globals'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +/** Upgrader for upgrading roles */ +export default async function ({ ncMeta }: NcUpgraderCtx) { + const users = await ncMeta.metaList2(null, null, MetaTable.USERS); + + for (const user of users) { + user.roles = user.roles + .split(',') + .map((r) => { + // update old role names with new roles + if (r === 'user') { + return OrgUserRoles.CREATOR; + } else if (r === 'user-new') { + return OrgUserRoles.VIEWER; + } + return r; + }) + .join(','); + await ncMeta.metaUpdate( + null, + null, + MetaTable.USERS, + { roles: user.roles }, + user.id, + ); + } + + // set invite only signup if user have environment variable set + if (process.env.NC_INVITE_ONLY_SIGNUP) { + await Store.saveOrUpdate( + { + value: '{ "invite_only_signup": true }', + key: NC_APP_SETTINGS, + }, + ncMeta, + ); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncProjectUpgraderV2_0090000.ts b/packages/nocodb-nest/src/version-upgrader/ncProjectUpgraderV2_0090000.ts new file mode 100644 index 0000000000..72ba347454 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncProjectUpgraderV2_0090000.ts @@ -0,0 +1,1339 @@ +import { + ModelTypes, + substituteColumnAliasWithIdInFormula, + UITypes, + ViewTypes, +} from 'nocodb-sdk'; +import Model from '../models/Model'; +import ProjectUser from '../models/ProjectUser'; +import Project from '../models/Project'; +import User from '../models/User'; +import Noco from '../Noco'; +import Column from '../models/Column'; +import NcHelp from '../utils/NcHelp'; +import View from '../models/View'; +import Sort from '../models/Sort'; +import Filter from '../models/Filter'; +import ModelRoleVisibility from '../models/ModelRoleVisibility'; +import { MetaTable } from '../utils/globals'; +import Hook from '../models/Hook'; +import FormViewColumn from '../models/FormViewColumn'; +import GridViewColumn from '../models/GridViewColumn'; +import { getUniqueColumnAliasName } from '../helpers/getUniqueName'; +import NcProjectBuilderEE from './v1-legacy/NcProjectBuilder'; +import Audit from '../models/Audit'; +import type GalleryView from '../models/GalleryView'; +import type FormView from '../models/FormView'; +import type { ViewType } from 'nocodb-sdk'; +import type KanbanView from '../models/KanbanView'; +import type GridView from '../models/GridView'; +import type RollupColumn from '../models/RollupColumn'; +import type { ROLLUP_FUNCTIONS } from '../models/RollupColumn'; +import type LinkToAnotherRecordColumn from '../models/LinkToAnotherRecordColumn'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +export default async function (ctx: NcUpgraderCtx) { + const ncMeta = ctx.ncMeta; + + const projects = await ctx.ncMeta.projectList(); + + for (const project of projects) { + // const projectConfig = JSON.parse(project.config); + + const projectBuilder = new NcProjectBuilderEE( + { ncMeta: ctx.ncMeta } as any, + { workingEnv: '_noco' } as any, + project, + ); + + await projectBuilder.init(); + } + + const usersObj = await migrateUsers(ncMeta); + const projectsObj = await migrateProjects(ncMeta); + await migrateProjectUsers(projectsObj, usersObj, ncMeta); + const migrationCtx = await migrateProjectModels(ncMeta); + + await migrateUIAcl(migrationCtx, ncMeta); + await migrateSharedViews(migrationCtx, ncMeta); + await migrateSharedBase(ncMeta); + await migratePlugins(ncMeta); + await migrateWebhooks(migrationCtx, ncMeta); + await migrateAutitLog(migrationCtx, projectsObj, ncMeta); +} + +async function migrateUsers(ncMeta = Noco.ncMeta) { + const users = await ncMeta.metaList(null, null, 'xc_users'); + const userObj: { [id: string]: User } = {}; + + for (const user of users) { + const user1 = await User.insert(user, ncMeta); + userObj[user1.id] = user1; + } + return userObj; +} + +async function migrateProjects( + ncMeta = Noco.ncMeta, +): Promise<{ [projectId: string]: Project }> { + const projects = await ncMeta.projectList(); + const projectsObj: { [projectId: string]: Project } = {}; + + for (const project of projects) { + const projectConfig = JSON.parse(project.config); + + const projectBody = { + id: project.id, + prefix: projectConfig.prefix, + is_meta: !!projectConfig.prefix, + title: projectConfig?.title, + bases: projectConfig?.envs?._noco?.db?.map((d) => { + const inflection = (d && d.meta && d.meta.inflection) || {}; + return { + is_meta: !!projectConfig.prefix, + type: d.client, + config: d, + inflection_column: inflection.cn, + inflection_table: inflection.tn, + }; + }), + }; + const p = await Project.createProject(projectBody, ncMeta); + projectsObj[p.id] = p; + } + return projectsObj; +} + +async function migrateProjectUsers( + projectsObj: { [p: string]: Project }, + usersObj: { [p: string]: User }, + ncMeta = Noco.ncMeta, +) { + const projectUsers = await ncMeta.metaList(null, null, 'nc_projects_users'); + + for (const projectUser of projectUsers) { + // skip if project is missing + if (!(projectUser.project_id in projectsObj)) continue; + + // skip if user is missing + if (!(projectUser.user_id in usersObj)) continue; + + await ProjectUser.insert( + { + project_id: projectUser.project_id, + fk_user_id: projectUser.user_id, + roles: projectUser.roles, + }, + ncMeta, + ); + } +} + +export interface ShowFieldsv1 { + [columnAlias: string]: boolean; +} + +export interface ViewStatusv1 { + type: string; +} + +export interface ColumnsWidthv1 { + [columnAlias: string]: string; +} + +export interface ExtraViewParamsv1 { + formParams?: { + name: string; + description: string; + submit: { + submitRedirectUrl?: string; + message: string; + showBlankForm: boolean; + showAnotherSubmit: boolean; + }; + emailMe: { + [email: string]: boolean; + }; + fields: { + [columnAlias: string]: { + help: string; + required: boolean; + label?: string; + description?: string; + }; + }; + }; +} + +export interface QueryParamsv1 { + filters: Array<{ + field: string; + op: string; + value: string | boolean | number; + logicOp: 'and' | 'or'; + }>; + sortList: Array<{ + field: string; + order: '' | '-'; + }>; + showFields: ShowFieldsv1; + fieldsOrder: string[]; + viewStatus: ViewStatusv1; + columnsWidth: ColumnsWidthv1; + extraViewParams: ExtraViewParamsv1; + showSystemFields: boolean; + coverImageField: string; +} + +interface Rollupv1 { + _cn: string; + rl: { + type: string; + tn: string; + cn: string; + vtn: string; + vrcn: string; + rcn: string; + rtn: string; + vcn: string; + _rtn: string; + rltn: string; + _rltn: string; + rlcn: string; + _rlcn: string; + fn: string; + }; +} + +interface Formulav1 { + _cn: string; + formula: { + value: string; + tree: any; + error: string[] | string; + }; +} + +interface Lookupv1 { + _cn: string; + lk: { + type: string; + tn: string; + cn: string; + vtn: string; + vrcn: string; + rcn: string; + rtn: string; + vcn: string; + _rtn: string; + ltn: string; + _ltn: string; + lcn: string; + _lcn: string; + }; +} + +interface LinkToAnotherRecordv1 { + _cn: string; + hm?: any; + bt?: any; + mm?: any; +} + +interface ModelMetav1 { + id: number | string; + project_id: string; + db_alias: string; + title: string; + alias: string; + type: 'table' | 'vtable' | 'view'; + meta: string; + schema: string; + schema_previous: string; + services: string; + messages: string; + enabled: boolean; + parent_model_title: string; + show_as: 'grid' | 'gallery' | 'form'; + query_params: string; + list_idx: number; + tags: string; + pinned: boolean; + mm: boolean; + m_to_m_meta: string; + order: number; + view_order: number; +} + +type ObjModelColumnRefv1 = { + [projectId: string]: { + [tableName: string]: { + [columnName: string]: Column; + }; + }; +}; +type ObjModelColumnAliasRefv1 = { + [projectId: string]: { + [tableName: string]: { + [columnAlias: string]: Column; + }; + }; +}; + +type ObjModelRefv1 = { + [projectId: string]: { + [tableName: string]: Model; + }; +}; +type ObjViewRefv1 = { + [projectId: string]: { + [tableName: string]: { + [viewName: string]: View; + }; + }; +}; +type ObjViewQPRefv1 = { + [projectId: string]: { + [tableName: string]: { + [viewName: string]: QueryParamsv1; + }; + }; +}; + +interface MigrateCtxV1 { + views: ModelMetav1[]; + objModelRef: ObjModelRefv1; + objModelAliasRef: ObjModelRefv1; + objModelColumnRef: ObjModelColumnRefv1; + objModelColumnAliasRef: ObjModelColumnAliasRefv1; + objViewRef: ObjViewRefv1; + objViewQPRef: ObjViewQPRefv1; + metas: ModelMetav1[]; +} + +// @ts-ignore +const filterV1toV2CompOpMap = { + 'is like': 'like', + '>': 'gt', + '<': 'lt', + '>=': 'gte', + '<=': 'lte', + 'is equal': 'eq', + 'is not null': 'notnull', + 'is null': 'null', + 'is not equal': 'neq', + 'is not like': 'nlike', +}; + +interface Relationv1 { + project_id?: string; + db_alias?: string; + tn?: string; + rtn?: string; + _tn?: string; + _rtn?: string; + cn?: string; + rcn?: string; + _cn?: string; + _rcn?: string; + referenced_db_alias?: string; + type?: string; + db_type?: string; + ur?: string; + dr?: string; +} + +async function migrateProjectModels( + ncMeta = Noco.ncMeta, +): Promise { + // @ts-ignore + + const metas: ModelMetav1[] = await ncMeta.metaList(null, null, 'nc_models'); + // @ts-ignore + const relations: Relationv1[] = await ncMeta.metaList( + null, + null, + 'nc_relations', + ); + const models: Model[] = []; + + // variable for keeping all + const objModelRef: ObjModelRefv1 = {}; + const objModelAliasRef: ObjModelRefv1 = {}; + const objModelColumnRef: ObjModelColumnRefv1 = {}; + const objModelColumnAliasRef: ObjModelColumnAliasRefv1 = {}; + const objViewRef: ObjViewRefv1 = {}; + const objViewQPRef: ObjViewQPRefv1 = {}; + + const virtualRelationColumnInsert: Array<() => Promise> = []; + const virtualColumnInsert: Array<() => Promise> = []; + const defaultViewsUpdate: Array<() => Promise> = []; + const views: ModelMetav1[] = []; + + for (const modelData of metas) { + // @ts-ignore + let queryParams: QueryParamsv1 = {}; + if (modelData.query_params) { + queryParams = JSON.parse(modelData.query_params); + } + + if (modelData.type === 'table' || modelData.type === 'view') { + // parse meta + + const project = await Project.getWithInfo(modelData.project_id, ncMeta); + + // skip if associated project is not found + if (!project) { + continue; + } + + const baseId = project.bases[0].id; + + const meta = JSON.parse(modelData.meta); + const model = await Model.insert( + project.id, + baseId, + { + order: modelData.order, + table_name: modelData.title, + title: modelData.alias, + // todo: sanitize + type: modelData.type === 'table' ? ModelTypes.TABLE : ModelTypes.VIEW, + mm: !!modelData.mm, + }, + ncMeta, + ); + models.push(model); + + const projectModelRefs = (objModelRef[project.id] = + objModelRef[project.id] || {}); + objModelRef[project.id][model.table_name] = model; + + objModelAliasRef[project.id] = objModelAliasRef[project.id] || {}; + objModelAliasRef[project.id][model.title] = model; + + const projectModelColumnRefs = (objModelColumnRef[project.id] = + objModelColumnRef[project.id] || {}); + objModelColumnRef[project.id][model.table_name] = + objModelColumnRef[project.id][model.table_name] || {}; + const projectModelColumnAliasRefs = (objModelColumnAliasRef[project.id] = + objModelColumnAliasRef[project.id] || {}); + objModelColumnAliasRef[project.id][model.table_name] = + objModelColumnAliasRef[project.id][model.table_name] || {}; + + objViewRef[project.id] = objViewRef[project.id] || {}; + objViewRef[project.id][modelData.title] = + objViewRef[project.id][modelData.title] || {}; + + objViewQPRef[project.id] = objViewQPRef[project.id] || {}; + objViewQPRef[project.id][modelData.title] = + objViewQPRef[project.id][modelData.title] || {}; + + // migrate table columns + for (const columnMeta of meta.columns) { + let system = false; + + if (meta.belongsTo?.find((bt) => bt.cn === columnMeta.cn)) { + system = true; + columnMeta.uidt = UITypes.ForeignKey; + } + + if (columnMeta.uidt === UITypes.Rating) { + columnMeta.uidt = UITypes.Number; + } + + const column = await Column.insert( + { + ...columnMeta, + title: columnMeta._cn, + column_name: columnMeta.cn, + system, + fk_model_id: model.id, + }, + ncMeta, + ); + + projectModelColumnRefs[model.table_name][column.column_name] = + projectModelColumnAliasRefs[model.table_name][column.title] = column; + } + + // migrate table virtual columns + for (const _columnMeta of meta.v) { + if (_columnMeta.mm || _columnMeta.hm || _columnMeta.bt) { + const columnMeta: LinkToAnotherRecordv1 = _columnMeta; + virtualRelationColumnInsert.push(async () => { + const rel = columnMeta.hm || columnMeta.bt || columnMeta.mm; + + const rel_column_id = + projectModelColumnRefs?.[rel.tn]?.[rel.cn]?.id; + + const tnId = projectModelRefs?.[rel.tn]?.id; + + const ref_rel_column_id = + projectModelColumnRefs?.[rel.rtn]?.[rel.rcn]?.id; + + const rtnId = projectModelRefs?.[rel.rtn]?.id; + + let fk_mm_model_id; + let fk_mm_child_column_id; + let fk_mm_parent_column_id; + + if (columnMeta.mm) { + fk_mm_model_id = projectModelRefs[rel.vtn].id; + fk_mm_child_column_id = + projectModelColumnRefs[rel.vtn][rel.vcn].id; + fk_mm_parent_column_id = + projectModelColumnRefs[rel.vtn][rel.vrcn].id; + } + + let virtual = false; + if (columnMeta.mm) { + const relation = relations.find( + (r) => + r.rtn === columnMeta.mm.tn && + r.rcn === columnMeta.mm.cn && + r.tn === columnMeta.mm.vtn && + r.cn === columnMeta.mm.vcn, + ); + virtual = relation?.type === 'virtual'; + } else if (columnMeta.hm) { + virtual = + relations.find( + (r) => + r.rtn === columnMeta.hm.rtn && + r.tn === columnMeta.hm.tn && + r.rcn === columnMeta.hm.rcn && + r.cn === columnMeta.hm.cn, + )?.type === 'virtual'; + } else if (columnMeta.bt) { + virtual = + relations.find( + (r) => + r.rtn === columnMeta.bt.rtn && + r.tn === columnMeta.bt.tn && + r.rcn === columnMeta.bt.rcn && + r.cn === columnMeta.bt.cn, + )?.type === 'virtual'; + } + + const column = await Column.insert( + { + project_id: project.id, + db_alias: baseId, + fk_model_id: model.id, + // cn: columnMeta.cn, + _cn: columnMeta._cn, + uidt: UITypes.LinkToAnotherRecord, + type: columnMeta.hm ? 'hm' : columnMeta.mm ? 'mm' : 'bt', + fk_child_column_id: rel_column_id, + fk_parent_column_id: ref_rel_column_id, + fk_index_name: rel.fkn, + ur: rel.ur, + dr: rel.dr, + fk_mm_model_id, + fk_mm_child_column_id, + fk_mm_parent_column_id, + fk_related_model_id: columnMeta.hm ? tnId : rtnId, + virtual, + }, + ncMeta, + ); + + projectModelColumnAliasRefs[model.table_name][column.title] = + column; + }); + } else { + // other virtual columns insert + virtualColumnInsert.push(async () => { + // migrate lookup column + if (_columnMeta.lk) { + const columnMeta: Lookupv1 = _columnMeta; + + const colBody: any = { + _cn: columnMeta._cn, + }; + + colBody.fk_lookup_column_id = + projectModelColumnRefs[columnMeta.lk.ltn][columnMeta.lk.lcn].id; + + const columns = Object.values( + projectModelColumnAliasRefs[model.table_name], + ); + + // extract related(virtual relation) column id + for (const col of columns) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = + await col.getColOptions(ncMeta); + if ( + colOpt.type === columnMeta.lk.type && + colOpt.fk_child_column_id === + projectModelColumnRefs[columnMeta.lk.tn][columnMeta.lk.cn] + .id && + colOpt.fk_parent_column_id === + projectModelColumnRefs[columnMeta.lk.rtn][ + columnMeta.lk.rcn + ].id && + (colOpt.type !== 'mm' || + colOpt.fk_mm_model_id === + projectModelRefs[columnMeta.lk.vtn].id) + ) { + colBody.fk_relation_column_id = col.id; + break; + } + } + } + + if (!colBody.fk_relation_column_id) { + throw new Error('relation not found'); + } + + const column = await Column.insert( + { + uidt: UITypes.Lookup, + ...colBody, + fk_model_id: model.id, + }, + ncMeta, + ); + projectModelColumnAliasRefs[model.table_name][column.title] = + column; + } else if (_columnMeta.rl) { + // migrate rollup column + const columnMeta: Rollupv1 = _columnMeta; + + const colBody: Partial = { + title: columnMeta._cn, + rollup_function: columnMeta.rl + .fn as (typeof ROLLUP_FUNCTIONS)[number], + }; + + colBody.fk_rollup_column_id = + projectModelColumnRefs[columnMeta.rl.rltn][ + columnMeta.rl.rlcn + ].id; + + const columns = Object.values( + projectModelColumnAliasRefs[model.table_name], + ); + + // extract related(virtual relation) column id + for (const col of columns) { + if (col.uidt === UITypes.LinkToAnotherRecord) { + const colOpt = + await col.getColOptions(ncMeta); + if ( + colOpt.type === columnMeta.rl.type && + colOpt.fk_child_column_id === + projectModelColumnRefs[columnMeta.rl.tn][columnMeta.rl.cn] + .id && + colOpt.fk_parent_column_id === + projectModelColumnRefs[columnMeta.rl.rtn][ + columnMeta.rl.rcn + ].id && + (colOpt.type !== 'mm' || + colOpt.fk_mm_model_id === + projectModelRefs[columnMeta.rl.vtn].id) + ) { + colBody.fk_relation_column_id = col.id; + break; + } + } + } + + if (!colBody.fk_relation_column_id) { + throw new Error('relation not found'); + } + const column = await Column.insert( + { + uidt: UITypes.Rollup, + ...colBody, + fk_model_id: model.id, + }, + ncMeta, + ); + projectModelColumnAliasRefs[model.table_name][column.title] = + column; + } else if (_columnMeta.formula) { + const columnMeta: Formulav1 = _columnMeta; + // migrate formula column + const colBody: any = { + _cn: columnMeta._cn, + }; + if (columnMeta?.formula?.error?.length) { + colBody.error = Array.isArray(columnMeta.formula.error) + ? columnMeta.formula.error.join(',') + : columnMeta.formula.error; + } else { + try { + colBody.formula = await substituteColumnAliasWithIdInFormula( + columnMeta.formula.value, + await model.getColumns(ncMeta), + ); + } catch { + colBody.error = 'Invalid formula'; + } + } + colBody.formula_raw = columnMeta.formula.value; + const column = await Column.insert( + { + uidt: UITypes.Formula, + ...colBody, + fk_model_id: model.id, + }, + ncMeta, + ); + + projectModelColumnAliasRefs[model.table_name][column.title] = + column; + } + }); + } + } + + // extract system hasmany relation + const hmColumns = meta.hasMany?.filter( + (hm) => + !meta.v.find( + (v) => + v.hm && + v.hm.rtn === hm.rtn && + v.hm.rcn === hm.rcn && + v.hm.tn === hm.tn && + v.hm.cn === hm.cn, + ), + ); + + for (const rel of hmColumns) { + virtualRelationColumnInsert.push(async () => { + const rel_column_id = projectModelColumnRefs?.[rel.tn]?.[rel.cn]?.id; + + const tnId = projectModelRefs?.[rel.tn]?.id; + + const ref_rel_column_id = + projectModelColumnRefs?.[rel.rtn]?.[rel.rcn]?.id; + + // const rtnId = projectModelRefs?.[rel.rtn]?.id; + + const virtual = + relations.find( + (r) => + r.rtn === rel.rtn && + r.tn === rel.tn && + r.rcn === rel.rcn && + r.cn === rel.cn, + )?.type === 'virtual'; + + const column = await Column.insert( + { + project_id: project.id, + db_alias: baseId, + fk_model_id: model.id, + // todo: populate unique name + _cn: getUniqueColumnAliasName([], `${rel.tn}List`), + uidt: UITypes.LinkToAnotherRecord, + type: 'hm', + fk_child_column_id: rel_column_id, + fk_parent_column_id: ref_rel_column_id, + fk_index_name: rel.fkn, + ur: rel.ur, + dr: rel.dr, + fk_related_model_id: tnId, + system: true, + virtual, + }, + ncMeta, + ); + + projectModelColumnAliasRefs[model.table_name][column.title] = column; + }); + } + + defaultViewsUpdate.push(async () => { + // insert default view data here + // @ts-ignore + const defaultView = await View.list(model.id, ncMeta).then( + (views) => views[0], + ); + + objViewRef[project.id][modelData.title][defaultView.title] = + defaultView; + objViewQPRef[project.id][modelData.title][defaultView.title] = + queryParams; + + const viewColumns = await View.getColumns(defaultView.id, ncMeta); + + const aliasColArr = Object.entries( + projectModelColumnAliasRefs[model.table_name], + ).sort(([a], [b]) => { + return ( + ((queryParams?.fieldsOrder || [])?.indexOf(a) + 1 || Infinity) - + ((queryParams?.fieldsOrder || [])?.indexOf(b) + 1 || Infinity) + ); + }); + let orderCount = 1; + for (const [_cn, column] of aliasColArr) { + const viewColumn = viewColumns.find( + (c) => column.id === c.fk_column_id, + ); + if (!viewColumn) continue; + await GridViewColumn.update( + viewColumn.id, + { + order: orderCount++, + show: queryParams?.showFields + ? queryParams?.showFields?.[_cn] || false + : true, + width: queryParams?.columnsWidth?.[_cn], + }, + ncMeta, + ); + } + await View.update( + defaultView.id, + { + show_system_fields: queryParams.showSystemFields, + order: modelData.view_order, + }, + ncMeta, + ); + }); + } else { + views.push(modelData); + } + } + + const type = Noco.getConfig()?.meta?.db?.client; + + await NcHelp.executeOperations(virtualRelationColumnInsert, type); + await NcHelp.executeOperations(virtualColumnInsert, type); + await NcHelp.executeOperations(defaultViewsUpdate, type); + + await migrateProjectModelViews( + { + metas, + views, + objModelRef, + objModelColumnAliasRef, + objModelColumnRef, + objViewRef, + objViewQPRef, + objModelAliasRef, + }, + ncMeta, + ); + + await migrateViewsParams( + { + metas, + views, + objModelRef, + objModelColumnAliasRef, + objModelColumnRef, + objViewRef, + objViewQPRef, + objModelAliasRef, + }, + ncMeta, + ); + + return { + metas, + views, + objModelRef, + objModelColumnAliasRef, + objModelColumnRef, + objViewRef, + objViewQPRef, + objModelAliasRef, + }; +} + +async function migrateProjectModelViews( + { + views, + objModelRef, + // objModelColumnRef, + objModelColumnAliasRef, + objViewRef, + objViewQPRef, + }: MigrateCtxV1, + ncMeta, +) { + for (const viewData of views) { + const project = await Project.getWithInfo(viewData.project_id, ncMeta); + // @ts-ignore + const baseId = project.bases[0].id; + + // @ts-ignore + let queryParams: QueryParamsv1 = {}; + if (viewData.query_params) { + queryParams = JSON.parse(viewData.query_params); + } + + objViewQPRef[project.id][viewData.parent_model_title][viewData.title] = + queryParams; + + const insertObj: Partial< + View & GridView & KanbanView & FormView & GalleryView + > = { + title: viewData.title, + show: true, + order: viewData.view_order, + fk_model_id: objModelRef[project.id][viewData.parent_model_title].id, + project_id: project.id, + base_id: baseId, + }; + + if (viewData.show_as === 'grid') { + insertObj.type = ViewTypes.GRID; + } else if (viewData.show_as === 'gallery') { + insertObj.type = ViewTypes.GALLERY; + insertObj.fk_cover_image_col_id = + objModelColumnAliasRef[project.id][viewData.parent_model_title][ + queryParams.coverImageField + ]?.id; + } else if (viewData.show_as === 'form') { + insertObj.type = ViewTypes.FORM; + insertObj.heading = queryParams.extraViewParams?.formParams?.name; + insertObj.subheading = + queryParams.extraViewParams?.formParams?.description; + insertObj.success_msg = + queryParams.extraViewParams?.formParams?.submit?.message; + insertObj.redirect_url = + queryParams.extraViewParams?.formParams?.submit?.submitRedirectUrl; + insertObj.email = JSON.stringify( + queryParams.extraViewParams?.formParams?.emailMe, + ); + insertObj.submit_another_form = + queryParams.extraViewParams?.formParams?.submit.showAnotherSubmit; + insertObj.show_blank_form = + queryParams.extraViewParams?.formParams?.submit?.showBlankForm; + } else throw new Error('not implemented'); + + const view = await View.insert(insertObj, ncMeta); + objViewRef[project.id][viewData.parent_model_title][view.title] = view; + + const viewColumns = await View.getColumns(view.id, ncMeta); + + const aliasColArr = Object.entries( + objModelColumnAliasRef[project.id][viewData.parent_model_title], + ).sort(([a], [b]) => { + return ( + ((queryParams?.fieldsOrder || [])?.indexOf(a) + 1 || Infinity) - + ((queryParams?.fieldsOrder || [])?.indexOf(b) + 1 || Infinity) + ); + }); + + let orderCount = 1; + + for (const [_cn, column] of aliasColArr) { + const viewColumn = viewColumns.find((c) => column.id === c.fk_column_id); + const order = orderCount++; + const show = queryParams?.showFields + ? queryParams?.showFields?.[_cn] || false + : true; + if (viewData.show_as === 'form') { + const columnParams = + queryParams?.extraViewParams?.formParams?.fields?.[_cn]; + await FormViewColumn.update( + viewColumn.id, + { + help: columnParams?.help?.slice(0, 254), + label: columnParams?.label, + required: columnParams?.required, + description: columnParams?.description, + order, + show, + }, + ncMeta, + ); + } else if (viewData.show_as === 'grid') { + const viewColumn = viewColumns.find( + (c) => column.id === c.fk_column_id, + ); + if (!viewColumn) continue; + await GridViewColumn.update( + viewColumn.id, + { + order, + show, + width: queryParams?.columnsWidth?.[_cn], + }, + ncMeta, + ); + } else { + await View.updateColumn( + view.id, + viewColumn.id, + { + order, + show, + }, + ncMeta, + ); + } + } + await View.update( + view.id, + { + show_system_fields: queryParams.showSystemFields, + order: viewData.view_order, + }, + ncMeta, + ); + } +} + +// migrate sort & filter +async function migrateViewsParams( + { + objModelColumnAliasRef, + objViewRef, + objViewQPRef, + objModelColumnRef, + }: MigrateCtxV1, + ncMeta, +) { + for (const projectId of Object.keys(objViewRef)) { + for (const tn of Object.keys(objViewRef[projectId])) { + for (const [viewTitle, view] of Object.entries( + objViewRef[projectId][tn], + )) { + const queryParams = objViewQPRef[projectId][tn][viewTitle]; + + if ( + queryParams?.viewStatus?.type && + queryParams?.viewStatus?.type !== view.lock_type + ) { + await View.update( + view.id, + { + lock_type: queryParams?.viewStatus?.type, + }, + ncMeta, + ); + view.lock_type = queryParams?.viewStatus + ?.type as ViewType['lock_type']; + } + // migrate view sort list + for (const sort of queryParams.sortList || []) { + await Sort.insert( + { + fk_column_id: sort.field + ? ( + objModelColumnAliasRef[projectId]?.[tn]?.[sort.field] || + objModelColumnRef[projectId]?.[tn]?.[sort.field] + )?.id || null + : null, + fk_view_id: view.id, + direction: sort.order === '-' ? 'desc' : 'asc', + }, + ncMeta, + ); + } + + // migrate view filter list + for (const filter of queryParams.filters || []) { + await Filter.insert( + { + fk_column_id: filter.field + ? ( + objModelColumnAliasRef?.[projectId]?.[tn]?.[filter.field] || + objModelColumnRef?.[projectId]?.[tn]?.[filter.field] + )?.id || null + : null, + fk_view_id: view.id, + comparison_op: filterV1toV2CompOpMap[filter.op], + logical_op: filter.logicOp, + value: filter.value, + }, + ncMeta, + ); + } + } + } + } +} + +async function migrateUIAcl(ctx: MigrateCtxV1, ncMeta: any) { + const uiAclList: Array<{ + role: string; + title: string; + type: 'vtable' | 'table' | 'view'; + disabled: boolean; + tn: string; + parent_model_title: string; + project_id: string; + }> = await ncMeta.metaList(null, null, 'nc_disabled_models_for_role'); + + for (const acl of uiAclList) { + // if missing model name skip the view acl migration + if (!acl?.title) continue; + + let fk_view_id; + if (acl.type === 'vtable') { + // if missing parent model name skip the view acl migration + if (!acl.parent_model_title) continue; + fk_view_id = + ctx.objViewRef[acl.project_id]?.[ + ( + ctx.objModelRef?.[acl.project_id]?.[acl.parent_model_title] || + ctx.objModelAliasRef?.[acl.project_id]?.[acl.parent_model_title] + )?.table_name + ]?.[acl.title]?.id; + } else { + fk_view_id = + ctx.objViewRef?.[acl.project_id]?.[acl.title]?.[ + ctx.objModelRef?.[acl.project_id]?.[acl.title]?.title + ].id || ctx.objViewRef[acl.project_id]?.[acl.title]?.[acl.title]?.id; + } + + // if view id missing skip ui acl view migration + if (!fk_view_id) continue; + + await ModelRoleVisibility.insert( + { + role: acl.role, + fk_view_id, + disabled: acl.disabled, + }, + ncMeta, + ); + } +} + +async function migrateSharedViews(ctx: MigrateCtxV1, ncMeta: any) { + const sharedViews: Array<{ + model_name: string; + view_type: 'vtable' | 'table' | 'view'; + view_id: string; + password: string; + view_name: string; + project_id: string; + }> = await ncMeta.metaList(null, null, 'nc_shared_views'); + + for (const sharedView of sharedViews) { + let fk_view_id; + + // if missing view name or model name skip the shared view migration + if (!sharedView.view_name || !sharedView.model_name) continue; + + if (sharedView.view_type !== 'table' && sharedView.view_type !== 'view') { + fk_view_id = + ctx.objViewRef[sharedView.project_id]?.[ + ( + ctx.objModelRef?.[sharedView.project_id]?.[sharedView.model_name] || + ctx.objModelAliasRef?.[sharedView.project_id]?.[ + sharedView.model_name + ] + )?.title + ]?.[sharedView.view_name]?.id; + } else { + fk_view_id = + ctx.objViewRef[sharedView.project_id]?.[sharedView.model_name]?.[ + ctx.objModelRef[sharedView.project_id]?.[sharedView.model_name]?.title + ]?.id || + ctx.objViewRef[sharedView.project_id]?.[sharedView.model_name]?.[ + sharedView.model_name + ]?.id; + } + + // if view id missing skip shared view migration + if (!fk_view_id) continue; + + await View.update( + fk_view_id, + { + uuid: sharedView.view_id, + password: sharedView.password, + }, + ncMeta, + ); + } +} + +async function migrateSharedBase(ncMeta: any) { + const sharedBases: Array<{ + roles: string; + shared_base_id: string; + enabled: boolean; + project_id: string; + password: string; + }> = await ncMeta.metaList(null, null, 'nc_shared_bases'); + + for (const sharedBase of sharedBases) { + await Project.update( + sharedBase.project_id, + { + uuid: sharedBase.shared_base_id, + password: sharedBase.password, + roles: sharedBase.roles, + }, + ncMeta, + ); + } +} + +async function migratePlugins(ncMeta: any) { + const plugins: Array = await ncMeta.metaList(null, null, 'nc_plugins'); + + for (const plugin of plugins) { + await ncMeta.metaInsert2(null, null, MetaTable.PLUGIN, { + title: plugin.title, + description: plugin.description, + active: plugin.active, + version: plugin.version, + docs: plugin.docs, + status: plugin.status, + status_details: plugin.status_details, + logo: plugin.logo, + tags: plugin.tags, + category: plugin.category, + input: plugin.input, + input_schema: plugin.input_schema, + creator: plugin.creator, + creator_website: plugin.creator_website, + price: plugin.price, + }); + } +} + +async function migrateWebhooks(ctx: MigrateCtxV1, ncMeta: any) { + const hooks: Array<{ + project_id: string; + db_alias: string; + title: string; + description: string; + env: string; + tn: string; + type: string; + event: 'after' | 'before'; + operation: 'delete' | 'update' | 'insert'; + async: boolean; + payload: string; + url: string; + headers: string; + condition: string; + notification: string; + retries: number; + retry_interval: number; + timeout: number; + active: boolean; + }> = await ncMeta.metaList(null, null, 'nc_hooks'); + + for (const hookMeta of hooks) { + if ( + !hookMeta.project_id || + !ctx.objModelRef[hookMeta?.project_id]?.[hookMeta?.tn] + ) { + continue; + } + const hook = await Hook.insert( + { + fk_model_id: ctx.objModelRef[hookMeta.project_id][hookMeta.tn].id, + project_id: hookMeta.project_id, + title: hookMeta.title, + description: hookMeta.description, + env: hookMeta.env, + type: hookMeta.type, + event: hookMeta.event, + operation: hookMeta.operation, + async: hookMeta.async, + payload: hookMeta.payload, + url: hookMeta.url, + headers: hookMeta.headers, + condition: !!hookMeta.condition, + notification: hookMeta.notification, + retries: hookMeta.retries, + retry_interval: hookMeta.retry_interval, + timeout: hookMeta.timeout, + active: hookMeta.active, + }, + ncMeta, + ); + let filters = []; + try { + filters = JSON.parse(hookMeta.condition); + } catch {} + // migrate view filter list + for (const filter of filters || []) { + await Filter.insert( + { + fk_column_id: filter.field + ? ( + ctx.objModelColumnRef[hookMeta.project_id][hookMeta.tn][ + filter.field + ] || + ctx.objModelColumnAliasRef[hookMeta.project_id][hookMeta.tn][ + filter.field + ] + ).id + : null, + fk_hook_id: hook.id, + logical_op: filter.logicOp, + comparison_op: filterV1toV2CompOpMap[filter.op], + value: filter.value, + }, + ncMeta, + ); + } + } +} + +async function migrateAutitLog( + ctx: MigrateCtxV1, + projectsObj: { [projectId: string]: Project }, + ncMeta: any, +) { + const audits: Array<{ + user: string; + ip: string; + project_id: string; + db_alias: string; + model_name: string; + model_id: string; + op_type: string; + op_sub_type: string; + status: string; + description: string; + details: string; + }> = await ncMeta.metaList(null, null, 'nc_audit'); + + for (const audit of audits) { + // skip deleted projects audit + if (!(audit.project_id in projectsObj)) continue; + + const insertObj: any = { + user: audit.user, + ip: audit.ip, + project_id: audit.project_id, + row_id: audit.model_id, + op_type: audit.op_type, + op_sub_type: audit.op_sub_type, + status: audit.status, + description: audit.description, + details: audit.details, + }; + + if (audit.model_name) { + const model = + ctx.objModelAliasRef?.[audit.project_id]?.[audit.model_name] || + ctx.objModelRef?.[audit.project_id]?.[audit.model_name] || + // extract model by using model_id property from audit + ctx.objModelRef?.[audit.project_id]?.[ + ctx.metas?.find((m) => m.id == audit.model_id)?.title + ] || + ctx.objModelAliasRef?.[audit.project_id]?.[ + ctx.metas?.find((m) => m.id == audit.model_id)?.alias + ]; + + // if model is not found skip audit insertion + if (!model) continue; + + insertObj.fk_model_id = model.id; + } + + await Audit.insert(insertObj, ncMeta); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncStickyColumnUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/ncStickyColumnUpgrader.ts new file mode 100644 index 0000000000..ce9a3ececd --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncStickyColumnUpgrader.ts @@ -0,0 +1,134 @@ +import { MetaTable } from '../utils/globals'; +import type { NcUpgraderCtx } from './NcUpgrader'; + +// before 0.104.3, display value column can be in any position in table +// with this upgrade we introduced sticky primary column feature +// this upgrader will make display value column first column in grid views + +export default async function ({ ncMeta }: NcUpgraderCtx) { + const grid_columns = await ncMeta.metaList2( + null, + null, + MetaTable.GRID_VIEW_COLUMNS, + ); + const grid_views = [...new Set(grid_columns.map((col) => col.fk_view_id))]; + + for (const view_id of grid_views) { + // get a list of view columns sorted by order + const view_columns = await ncMeta.metaList2( + null, + null, + MetaTable.GRID_VIEW_COLUMNS, + { + condition: { + fk_view_id: view_id, + }, + orderBy: { + order: 'asc', + }, + }, + ); + const view_columns_meta = []; + + // get column meta for each view column + for (const col of view_columns) { + const col_meta = await ncMeta.metaGet(null, null, MetaTable.COLUMNS, { + id: col.fk_column_id, + }); + view_columns_meta.push(col_meta); + } + + // if no display value column is set + if (!view_columns_meta.some((column) => column.pv)) { + const pkIndex = view_columns_meta.findIndex((column) => column.pk); + + // if PK is at the end of table + if (pkIndex === view_columns_meta.length - 1) { + if (pkIndex > 0) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.COLUMNS, + { pv: true }, + view_columns_meta[pkIndex - 1].id, + ); + } else if (view_columns_meta.length > 0) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.COLUMNS, + { pv: true }, + view_columns_meta[0].id, + ); + } + // pk is not at the end of table + } else if (pkIndex > -1) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.COLUMNS, + { pv: true }, + view_columns_meta[pkIndex + 1].id, + ); + // no pk at all + } else if (view_columns_meta.length > 0) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.COLUMNS, + { pv: true }, + view_columns_meta[0].id, + ); + } + } + + const primary_value_column_meta = view_columns_meta.find((col) => col.pv); + + if (primary_value_column_meta) { + const primary_value_column = view_columns.find( + (col) => col.fk_column_id === primary_value_column_meta.id, + ); + const primary_value_column_index = view_columns.findIndex( + (col) => col.fk_column_id === primary_value_column_meta.id, + ); + const view_orders = view_columns.map((col) => col.order); + const view_min_order = Math.min(...view_orders); + + // if primary_value_column is not visible, make it visible + if (!primary_value_column.show) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.GRID_VIEW_COLUMNS, + { show: true }, + primary_value_column.id, + ); + } + + if ( + primary_value_column.order === view_min_order && + view_orders.filter((o) => o === view_min_order).length === 1 + ) { + // if primary_value_column is in first order do nothing + continue; + } else { + // if primary_value_column not in first order, move it to the start of array + if (primary_value_column_index !== 0) { + const temp_pv = view_columns.splice(primary_value_column_index, 1); + view_columns.unshift(...temp_pv); + } + + // update order of all columns in view to match the order in array + for (let i = 0; i < view_columns.length; i++) { + await ncMeta.metaUpdate( + null, + null, + MetaTable.GRID_VIEW_COLUMNS, + { order: i + 1 }, + view_columns[i].id, + ); + } + } + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/ncUpgradeErrors.ts b/packages/nocodb-nest/src/version-upgrader/ncUpgradeErrors.ts new file mode 100644 index 0000000000..3b9b7330ab --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/ncUpgradeErrors.ts @@ -0,0 +1,27 @@ +export function throwTimeoutError(e, timeoutErrorInfo) { + if ( + [ + 'EHOSTDOWN', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENOTFOUND', + 'ECONNREFUSED', + ].includes(e.code) + ) { + let db = ''; + if (timeoutErrorInfo.connection.filename) { + // for sqlite + db = timeoutErrorInfo.connection.filename; + } else if ( + timeoutErrorInfo.connection.database && + timeoutErrorInfo.connection.host && + timeoutErrorInfo.connection.port + ) { + db = `${timeoutErrorInfo.connection.database} (${timeoutErrorInfo.connection.host}:${timeoutErrorInfo.connection.port})`; + } + throw new Error( + `Failed to connect the database ${db} for Project ${timeoutErrorInfo.projectTitle}. + Please fix the connection issue or remove the project before trying to upgrade.` + ); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/BaseApiBuilder.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/BaseApiBuilder.ts new file mode 100644 index 0000000000..0cc17e05e2 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/BaseApiBuilder.ts @@ -0,0 +1,611 @@ +import debug from 'debug'; +import { Router } from 'express'; +import { BaseModelSql } from '../../db/BaseModelSql'; +import { XKnex } from '../../db/CustomKnex'; +import ModelXcMetaFactory from '../../db/sql-mgr/code/models/xc/ModelXcMetaFactory' +import { DbConfig, NcConfig } from '../../interface/config'; +import inflection from 'inflection'; +import { MetaService } from '../../meta/meta.service'; +import NcConnectionMgr from '../../utils/common/NcConnectionMgr'; +import ncModelsOrderUpgrader from './jobs/ncModelsOrderUpgrader'; +import ncParentModelTitleUpgrader from './jobs/ncParentModelTitleUpgrader'; +import ncRemoveDuplicatedRelationRows from './jobs/ncRemoveDuplicatedRelationRows'; +import type Noco from '../../Noco'; +import type NcProjectBuilder from './NcProjectBuilder'; +import type { MysqlClient, PgClient, SqlClient } from 'nc-help'; + +const log = debug('nc:api:base'); + +export default abstract class BaseApiBuilder { + public abstract readonly type: string; + + public get knex(): XKnex { + return this.sqlClient?.knex || this.dbDriver; + } + + public get prefix() { + return this.projectBuilder?.prefix; + } + + public get apiType(): string { + return this.connectionConfig?.meta?.api?.type; + } + + public get apiPrefix(): string { + return this.connectionConfig?.meta?.api?.prefix; + } + + public get dbAlias(): any { + return this.connectionConfig?.meta?.dbAlias; + } + + public get router(): Router { + if (!this.apiRouter) { + this.baseLog(`router : Initializing builder router`); + this.apiRouter = Router(); + // (this.app as any).router.use('/', this.apiRouter) + (this.projectBuilder as any).router.use('/', this.apiRouter); + } + return this.apiRouter; + } + + public get routeVersionLetter(): string { + return this.connectionConfig?.meta?.api?.prefix || 'v1'; + } + + protected get projectId(): string { + return this.projectBuilder?.id; + } + + public get xcModels() { + return this.models; + } + + public get client() { + return this.connectionConfig?.client; + } + + public readonly app: T; + + public hooks: { + [tableName: string]: { + [key: string]: Array<{ + event: string; + url: string; + [key: string]: any; + }>; + }; + }; + + public formViews: { + [tableName: string]: any; + }; + + protected tablesCount = 0; + protected relationsCount = 0; + protected viewsCount = 0; + protected functionsCount = 0; + protected proceduresCount = 0; + + protected projectBuilder: NcProjectBuilder; + + protected models: { [key: string]: BaseModelSql }; + + protected metas: { [key: string]: NcMetaData }; + + protected sqlClient: MysqlClient | PgClient | SqlClient | any; + + protected dbDriver: XKnex; + protected config: NcConfig; + protected connectionConfig: DbConfig; + + protected procedureOrFunctionAcls: { + [name: string]: { [role: string]: boolean }; + }; + protected xcMeta: MetaService; + + private apiRouter: Router; + + constructor( + app: T, + projectBuilder: NcProjectBuilder, + config: NcConfig, + connectionConfig: DbConfig, + ) { + this.models = {}; + this.app = app; + this.config = config; + this.connectionConfig = connectionConfig; + this.metas = {}; + this.procedureOrFunctionAcls = {}; + this.hooks = {}; + this.formViews = {}; + this.projectBuilder = projectBuilder; + } + + public getDbType(): any { + return this.connectionConfig.client; + } + + public getDbName(): any { + return (this.connectionConfig.connection as any)?.database; + } + + public getDbAlias(): any { + return this.connectionConfig?.meta?.dbAlias; + } + + public async getSqlClient() { + return NcConnectionMgr.getSqlClient({ + dbAlias: this.dbAlias, + env: this.config.env, + config: this.config, + projectId: this.projectId, + }); + } + + public async xcUpgrade(): Promise { + const NC_VERSIONS = [ + { name: '0009000', handler: null }, + { name: '0009044', handler: this.ncUpManyToMany.bind(this) }, + { name: '0083006', handler: ncModelsOrderUpgrader }, + { name: '0083007', handler: ncParentModelTitleUpgrader }, + { name: '0083008', handler: ncRemoveDuplicatedRelationRows }, + { name: '0084002', handler: this.ncUpAddNestedResolverArgs.bind(this) }, + ]; + if (!(await this.xcMeta?.knex?.schema?.hasTable?.('nc_store'))) { + return; + } + this.baseLog(`xcUpgrade : Getting configuration from meta database`); + + const config = await this.xcMeta.metaGet( + this.projectId, + this.dbAlias, + 'nc_store', + { key: 'NC_CONFIG' }, + ); + + if (config) { + const configObj: NcConfig = JSON.parse(config.value); + if (configObj.version !== process.env.NC_VERSION) { + for (const version of NC_VERSIONS) { + // compare current version and old version + if (version.name > configObj.version) { + this.baseLog( + `xcUpgrade : Upgrading '%s' => '%s'`, + configObj.version, + version.name, + ); + await version?.handler?.({ + xcMeta: this.xcMeta, + builder: this, + dbAlias: this.dbAlias, + projectId: this.projectId, + }); + + // update version in meta after each upgrade + configObj.version = version.name; + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_store', + { + value: JSON.stringify(configObj), + }, + { + key: 'NC_CONFIG', + }, + ); + + // todo: backup data + } + if (version.name === process.env.NC_VERSION) { + break; + } + } + configObj.version = process.env.NC_VERSION; + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_store', + { + value: JSON.stringify(configObj), + }, + { + key: 'NC_CONFIG', + }, + ); + } + } else { + this.baseLog(`xcUpgrade : Inserting config to meta database`); + const configObj: NcConfig = JSON.parse(JSON.stringify(this.config)); + delete configObj.envs; + const isOld = ( + await this.xcMeta.metaList(this.projectId, this.dbAlias, 'nc_models') + )?.length; + configObj.version = isOld ? '0009000' : process.env.NC_VERSION; + await this.xcMeta.metaInsert(this.projectId, this.dbAlias, 'nc_store', { + key: 'NC_CONFIG', + value: JSON.stringify(configObj), + }); + if (isOld) { + await this.xcUpgrade(); + } + } + } + + public getProjectId(): string { + return this.projectId; + } + + public async init(): Promise { + await this.xcUpgrade(); + } + + protected async initDbDriver(): Promise { + this.dbDriver = await NcConnectionMgr.get({ + dbAlias: this.dbAlias, + env: this.config.env, + config: this.config, + projectId: this.projectId, + }); + this.sqlClient = await NcConnectionMgr.getSqlClient({ + dbAlias: this.dbAlias, + env: this.config.env, + config: this.config, + projectId: this.projectId, + }); + } + + private baseLog(str, ...args): void { + log(`${this.dbAlias} : ${str}`, ...args); + } + + protected generateContextForTable( + tn: string, + columns: any[], + relations, + hasMany: any[], + belongsTo: any[], + type = 'table', + tableNameAlias?: string, + ): any { + this.baseLog(`generateContextForTable : '%s' %s`, tn, type); + + for (const col of columns) { + col._cn = col._cn || this.getColumnNameAlias(col); + } + + // tslint:disable-next-line:variable-name + const _tn = tableNameAlias || this.getTableNameAlias(tn); + + const ctx = { + dbType: this.connectionConfig.client, + tn, + _tn, + tn_camelize: inflection.camelize(_tn), + tn_camelize_low: inflection.camelize(_tn, true), + columns, + relations, + hasMany, + belongsTo, + dbAlias: '', + routeVersionLetter: this.routeVersionLetter, + type, + project_id: this.projectId, + }; + return ctx; + } + + private getColumnNameAlias(col, tableName?: string) { + return ( + this.metas?.[tableName]?.columns?.find((c) => c.cn === col.cn)?._cn || + col._cn || + this.getInflectedName(col.cn, this.connectionConfig?.meta?.inflection?.cn) + ); + } + + // table alias functions + protected getInflectedName(_name: string, inflectionFns: string): string { + let name = _name; + if (process.env.NC_INFLECTION) { + inflectionFns = 'camelize'; + } + + if (inflectionFns && inflectionFns !== 'none') { + name = inflectionFns + .split(',') + .reduce((out, fn) => inflection?.[fn]?.(out) || out, name); + } + return this.apiType === 'graphql' ? name.replace(/[^_\da-z]/gi, '_') : name; + } + + protected async ncUpAddNestedResolverArgs(_ctx: any): Promise {} + + protected getTableNameAlias(tableName: string) { + let tn = tableName; + if (this.metas?.[tn]?._tn) { + return this.metas?.[tn]?._tn; + } + + if (this.projectBuilder?.prefix) { + tn = tn.replace(this.projectBuilder?.prefix, ''); + } + + const modifiedTableName = tn?.replace(/^(?=\d+)/, 'ISN___'); + return this.getInflectedName( + modifiedTableName, + this.connectionConfig?.meta?.inflection?.tn, + ); + } + + protected async ncUpManyToMany(_ctx: any): Promise { + const models = await this.xcMeta.metaList( + this.projectId, + this.dbAlias, + 'nc_models', + { + fields: ['meta'], + condition: { + type: 'table', + }, + }, + ); + if (!models.length) { + return; + } + const metas = []; + // add virtual columns for relations + for (const metaObj of models) { + const meta = JSON.parse(metaObj.meta); + metas.push(meta); + const ctx = this.generateContextForTable( + meta.tn, + meta.columns, + [], + meta.hasMany, + meta.belongsTo, + meta.type, + meta._tn, + ); + // generate virtual columns + meta.v = ModelXcMetaFactory.create(this.connectionConfig, { + dir: '', + ctx, + filename: '', + }).getVitualColumns(); + // set default display values + ModelXcMetaFactory.create( + this.connectionConfig, + {}, + ).mapDefaultDisplayValue(meta.columns); + // update meta + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + meta: JSON.stringify(meta), + }, + { title: meta.tn }, + ); + } + + // generate many to many relations an columns + await this.getManyToManyRelations({ localMetas: metas }); + return metas; + } + + protected async getManyToManyRelations({ + parent = null, + child = null, + localMetas = null, + } = {}): Promise> { + const metas = new Set(); + const assocMetas = new Set(); + + if (localMetas) { + for (const meta of localMetas) { + this.metas[meta.tn] = meta; + } + } + + for (const meta of Object.values(this.metas)) { + // check if table is a Bridge table(or Associative Table) by checking + // number of foreign keys and columns + if (meta.belongsTo?.length === 2 && meta.columns.length < 5) { + if ( + parent && + child && + !( + [parent, child].includes(meta.belongsTo[0].rtn) && + [parent, child].includes(meta.belongsTo[1].rtn) + ) + ) { + continue; + } + + const tableMetaA = this.metas[meta.belongsTo[0].rtn]; + const tableMetaB = this.metas[meta.belongsTo[1].rtn]; + + /* // remove hasmany relation with associate table from tables + tableMetaA.hasMany.splice(tableMetaA.hasMany.findIndex(hm => hm.tn === meta.tn), 1) + tableMetaB.hasMany.splice(tableMetaB.hasMany.findIndex(hm => hm.tn === meta.tn), 1)*/ + + // add manytomany data under metadata of both linked tables + tableMetaA.manyToMany = tableMetaA.manyToMany || []; + if (tableMetaA.manyToMany.every((mm) => mm.vtn !== meta.tn)) { + tableMetaA.manyToMany.push({ + tn: tableMetaA.tn, + cn: meta.belongsTo[0].rcn, + vtn: meta.tn, + vcn: meta.belongsTo[0].cn, + vrcn: meta.belongsTo[1].cn, + rtn: meta.belongsTo[1].rtn, + rcn: meta.belongsTo[1].rcn, + _tn: tableMetaA._tn, + _cn: meta.belongsTo[0]._rcn, + _rtn: meta.belongsTo[1]._rtn, + _rcn: meta.belongsTo[1]._rcn, + }); + metas.add(tableMetaA); + } + // ignore if A & B are same table + if (tableMetaB !== tableMetaA) { + tableMetaB.manyToMany = tableMetaB.manyToMany || []; + if (tableMetaB.manyToMany.every((mm) => mm.vtn !== meta.tn)) { + tableMetaB.manyToMany.push({ + tn: tableMetaB.tn, + cn: meta.belongsTo[1].rcn, + vtn: meta.tn, + vcn: meta.belongsTo[1].cn, + vrcn: meta.belongsTo[0].cn, + rtn: meta.belongsTo[0].rtn, + rcn: meta.belongsTo[0].rcn, + _tn: tableMetaB._tn, + _cn: meta.belongsTo[1]._rcn, + _rtn: meta.belongsTo[0]._rtn, + _rcn: meta.belongsTo[0]._rcn, + }); + metas.add(tableMetaB); + } + } + assocMetas.add(meta); + } + } + + // Update metadata of tables which have manytomany relation + // and recreate basemodel with new meta information + for (const meta of metas) { + let queryParams; + + // update showfields on new many to many relation create + if (parent && child) { + try { + queryParams = JSON.parse( + ( + await this.xcMeta.metaGet( + this.projectId, + this.dbAlias, + 'nc_models', + { title: meta.tn }, + ) + ).query_params, + ); + } catch (e) { + // ignore + } + } + + meta.v = [ + ...meta.v.filter( + (vc) => !(vc.hm && meta.manyToMany.some((mm) => vc.hm.tn === mm.vtn)), + ), + // todo: ignore duplicate m2m relations + // todo: optimize, just compare associative table(Vtn) + ...meta.manyToMany + .filter( + (v, i) => + !meta.v.some( + (v1) => + v1.mm && + ((v1.mm.tn === v.tn && v.rtn === v1.mm.rtn) || + (v1.mm.rtn === v.tn && v.tn === v1.mm.rtn)) && + v.vtn === v1.mm.vtn, + ) && + // ignore duplicate + !meta.manyToMany.some( + (v1, i1) => + i1 !== i && + v1.tn === v.tn && + v.rtn === v1.rtn && + v.vtn === v1.vtn, + ), + ) + .map((mm) => { + if ( + queryParams?.showFields && + !(`${mm._tn} <=> ${mm._rtn}` in queryParams.showFields) + ) { + queryParams.showFields[`${mm._tn} <=> ${mm._rtn}`] = true; + } + + return { + mm, + _cn: `${mm._tn} <=> ${mm._rtn}`, + }; + }), + ]; + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + meta: JSON.stringify(meta), + ...(queryParams ? { query_params: JSON.stringify(queryParams) } : {}), + }, + { title: meta.tn }, + ); + // XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::')); + // if (!localMetas) { + // this.models[meta.tn] = this.getBaseModel(meta); + // } + } + + // Update metadata of associative table + for (const meta of assocMetas) { + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + mm: 1, + }, + { title: meta.tn }, + ); + } + + return metas; + } + + +} + +interface NcBuilderUpgraderCtx { + xcMeta: MetaService; + builder: BaseApiBuilder; + projectId: string; + dbAlias: string; +} + +interface NcMetaData { + tn: string; + _tn?: string; + v: Array<{ + _cn?: string; + [key: string]: any; + }>; + columns: Array<{ + _cn?: string; + cn?: string; + uidt?: string; + [key: string]: any; + }>; + + [key: string]: any; +} + +type XcTablesPopulateParams = { + tableNames?: Array<{ + tn: string; + _tn?: string; + }>; + type?: 'table' | 'view' | 'function' | 'procedure'; + columns?: { + [tn: string]: any; + }; + oldMetas?: { + [tn: string]: NcMetaData; + }; +}; +export { NcBuilderUpgraderCtx, NcMetaData, XcTablesPopulateParams }; diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/NcProjectBuilder.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/NcProjectBuilder.ts new file mode 100644 index 0000000000..9586810e13 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/NcProjectBuilder.ts @@ -0,0 +1,198 @@ +import { Router } from 'express'; +import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory'; +import Noco from '../../Noco'; +import { GqlApiBuilder } from './gql/GqlApiBuilder'; +import { RestApiBuilder } from './rest/RestApiBuilder'; +import type { NcConfig } from '../../interface/config'; + +export default class NcProjectBuilder { + public readonly id: string; + public readonly title: string; + public readonly description: string; + public readonly router: Router; + public readonly apiBuilders: Array = []; + private _config: any; + + protected startTime; + protected app: Noco; + protected appConfig: NcConfig; + protected apiInfInfoList: any[] = []; + protected aggregatedApiInfo: any; + protected authHook: any; + + constructor(app: Noco, appConfig: NcConfig, project: any) { + this.app = app; + this.appConfig = appConfig; + + if (project) { + this.id = project.id; + this.title = project.title; + this.description = project.description; + this._config = { ...this.appConfig, ...JSON.parse(project.config) }; + this.router = Router(); + } + } + + public async init(_isFirstTime?: boolean) { + try { + // await this.addAuthHookToMiddleware(); + + this.startTime = Date.now(); + const allRoutesInfo: any[] = []; + // await this.app.ncMeta.projectStatusUpdate(this.id, 'starting'); + // await this.syncMigration(); + await this._createApiBuilder(); + // this.initApiInfoRoute(); + + /* Create REST APIs / GraphQL Resolvers */ + for (const meta of this.apiBuilders) { + let routeInfo; + if (meta instanceof RestApiBuilder) { + console.log( + `Creating REST APIs ${meta.getDbType()} - > ${meta.getDbName()}`, + ); + routeInfo = await (meta as RestApiBuilder).init(); + } else if (meta instanceof GqlApiBuilder) { + console.log( + `Creating GraphQL APIs ${meta.getDbType()} - > ${meta.getDbName()}`, + ); + routeInfo = await (meta as GqlApiBuilder).init(); + } + allRoutesInfo.push(routeInfo); + // this.progress(routeInfo, allRoutesInfo, isFirstTime); + } + + // this.app.projectRouter.use(`/nc/${this.id}`, this.router); + // await this.app.ncMeta.projectStatusUpdate(this.id, 'started'); + } catch (e) { + console.log(e); + throw e; + // await this.app.ncMeta.projectStatusUpdate(this.id, 'stopped'); + } + } + + protected async _createApiBuilder() { + this.apiBuilders.splice(0, this.apiBuilders.length); + let i = 0; + + const connectionConfigs = []; + + /* for each db create an api builder */ + for (const db of this.config?.envs?.[this.appConfig?.workingEnv]?.db || + []) { + let Builder; + switch (db.meta.api.type) { + case 'graphql': + Builder = GqlApiBuilder; + break; + + case 'rest': + Builder = RestApiBuilder; + break; + } + + if ((db?.connection as any)?.database) { + const connectionConfig = { + ...db, + meta: { + ...db.meta, + api: { + ...db.meta.api, + prefix: db.meta.api.prefix || this.genVer(i), + }, + }, + }; + + this.apiBuilders.push( + new Builder( + this.app, + this, + this.config, + connectionConfig, + this.app.ncMeta, + ), + ); + connectionConfigs.push(connectionConfig); + i++; + } else if (db.meta?.allSchemas) { + /* get all schemas and create APIs for all of them */ + const sqlClient = await SqlClientFactory.create({ + ...db, + connection: { ...db.connection, database: undefined }, + }); + + const schemaList = (await sqlClient.schemaList({}))?.data?.list; + for (const schema of schemaList) { + const connectionConfig = { + ...db, + connection: { ...db.connection, database: schema.schema_name }, + meta: { + ...db.meta, + dbAlias: i ? db.meta.dbAlias + i : db.meta.dbAlias, + api: { + ...db.meta.api, + prefix: db.meta.api.prefix || this.genVer(i), + }, + }, + }; + + this.apiBuilders.push( + new Builder( + this.app, + this, + this.config, + connectionConfig, + this.app.ncMeta, + ), + ); + connectionConfigs.push(connectionConfig); + + i++; + } + + sqlClient.knex.destroy(); + } + } + if (this.config?.envs?.[this.appConfig.workingEnv]?.db) { + this.config.envs[this.appConfig.workingEnv].db.splice( + 0, + this.config.envs[this.appConfig.workingEnv].db.length, + ...connectionConfigs, + ); + } + } + + protected genVer(i): string { + const l = 'vwxyzabcdefghijklmnopqrstu'; + return ( + i + .toString(26) + .split('') + .map((v) => l[parseInt(v, 26)]) + .join('') + '1' + ); + } + + protected static triggerGarbageCollect() { + try { + if (global.gc) { + global.gc(); + } + } catch (e) { + console.log('`node --expose-gc index.js`'); + process.exit(); + } + } + + public get prefix(): string { + return this.config?.prefix; + } + + public get config(): any { + return this._config; + } + + public updateConfig(config: string) { + this._config = { ...this.appConfig, ...JSON.parse(config) }; + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/gql/GqlApiBuilder.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/gql/GqlApiBuilder.ts new file mode 100644 index 0000000000..4cab838253 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/gql/GqlApiBuilder.ts @@ -0,0 +1,149 @@ +import debug from 'debug'; +import { Router } from 'express'; +import GqlXcSchemaFactory from '../../../../../nocodb/src/lib/db/sql-mgr/code/gql-schema/xc-ts/GqlXcSchemaFactory'; +import { MetaService } from '../../../meta/meta.service'; +import Noco from '../../../Noco'; +import type NcProjectBuilder from '../NcProjectBuilder'; +import type { DbConfig, NcConfig } from '../../../interface/config'; +import type XcMetaMgr from '../../../interface/XcMetaMgr'; +import BaseApiBuilder from '../BaseApiBuilder'; + +export class GqlApiBuilder extends BaseApiBuilder implements XcMetaMgr { + public readonly type = 'gql'; + + private readonly gqlRouter: Router; + + constructor( + app: Noco, + projectBuilder: NcProjectBuilder, + config: NcConfig, + connectionConfig: DbConfig, + xcMeta?: MetaService, + ) { + super(app, projectBuilder, config, connectionConfig); + this.config = config; + this.connectionConfig = connectionConfig; + this.gqlRouter = Router(); + this.xcMeta = xcMeta; + } + + public async init(): Promise { + await super.init(); + } + + protected async ncUpAddNestedResolverArgs(_ctx: any): Promise { + const models = await this.xcMeta.metaList( + this.projectId, + this.dbAlias, + 'nc_models', + { + fields: ['meta'], + condition: { + type: 'table', + }, + }, + ); + if (!models.length) { + return; + } + // add virtual columns for relations + for (const metaObj of models) { + const meta = JSON.parse(metaObj.meta); + const ctx = this.generateContextForTable( + meta.tn, + meta.columns, + [], + meta.hasMany, + meta.belongsTo, + meta.type, + meta._tn, + ); + + /* generate gql schema of the table */ + const schema = GqlXcSchemaFactory.create(this.connectionConfig, { + dir: '', + ctx: { + ...ctx, + manyToMany: meta.manyToMany, + }, + filename: '', + }).getString(); + + /* update schema in metadb */ + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + schema, + }, + { + title: meta.tn, + type: 'table', + }, + ); + } + } + + protected async ncUpManyToMany(ctx: any): Promise { + const metas = await super.ncUpManyToMany(ctx); + + if (!metas) { + return; + } + for (const meta of metas) { + const ctx = this.generateContextForTable( + meta.tn, + meta.columns, + [], + meta.hasMany, + meta.belongsTo, + meta.type, + meta._tn, + ); + + /* generate gql schema of the table */ + const schema = GqlXcSchemaFactory.create(this.connectionConfig, { + dir: '', + ctx: { + ...ctx, + manyToMany: meta.manyToMany, + }, + filename: '', + }).getString(); + + /* update schema in metadb */ + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + schema, + }, + { + title: meta.tn, + type: 'table', + }, + ); + + // todo : add loaders + + if (meta.manyToMany) { + for (const mm of meta.manyToMany) { + await this.xcMeta.metaInsert( + this.projectId, + this.dbAlias, + 'nc_loaders', + { + title: `${mm.tn}Mm${mm.rtn}List`, + parent: mm.tn, + child: mm.rtn, + relation: 'mm', + resolver: 'mmlist', + }, + ); + } + } + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncModelsOrderUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncModelsOrderUpgrader.ts new file mode 100644 index 0000000000..602b44f91d --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncModelsOrderUpgrader.ts @@ -0,0 +1,48 @@ +import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder'; + +export default async function (ctx: NcBuilderUpgraderCtx) { + const models = await ctx.xcMeta.metaList( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + xcCondition: { + _or: [{ type: { eq: 'table' } }, { type: { eq: 'view' } }], + }, + }, + ); + let order = 0; + for (const model of models) { + await ctx.xcMeta.metaUpdate( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + order: ++order, + view_order: 1, + }, + model.id, + ); + + const views = await ctx.xcMeta.metaList( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + condition: { parent_model_title: model.title }, + }, + ); + let view_order = 1; + for (const view of views) { + await ctx.xcMeta.metaUpdate( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + view_order: ++view_order, + }, + view.id, + ); + } + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncParentModelTitleUpgrader.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncParentModelTitleUpgrader.ts new file mode 100644 index 0000000000..c57eaf1500 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncParentModelTitleUpgrader.ts @@ -0,0 +1,29 @@ +import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder'; + +export default async function (ctx: NcBuilderUpgraderCtx) { + const views = await ctx.xcMeta.metaList( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + condition: { + type: 'vtable', + }, + }, + ); + + for (const view of views) { + await ctx.xcMeta.metaUpdate( + ctx.projectId, + ctx.dbAlias, + 'nc_disabled_models_for_role', + { + parent_model_title: view.parent_model_title, + }, + { + type: 'vtable', + title: view.title, + }, + ); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncRemoveDuplicatedRelationRows.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncRemoveDuplicatedRelationRows.ts new file mode 100644 index 0000000000..2684150a27 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncRemoveDuplicatedRelationRows.ts @@ -0,0 +1,113 @@ +import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder'; + +export default async function (ctx: NcBuilderUpgraderCtx) { + try { + const relations = await ctx.xcMeta.metaList( + ctx.projectId, + ctx.dbAlias, + 'nc_relations', + ); + + const duplicates = []; + + for (const relation of relations) { + if (relation.type !== 'real' || duplicates.includes(relation)) continue; + const duplicateRelIndex = relations.findIndex( + (rel) => + rel !== relation && + rel.tn === relation.tn && + rel.rtn === relation.rtn && + rel.cn === relation.cn && + rel.rcn === relation.rcn && + rel.type === 'real', + ); + + if (duplicateRelIndex > -1) duplicates.push(relations[duplicateRelIndex]); + } + + // delete relation + for (const dupRelation of duplicates) { + await ctx.xcMeta.metaDelete( + ctx.projectId, + ctx.dbAlias, + 'nc_relations', + dupRelation.id, + ); + { + const tnModel = await ctx.xcMeta.metaGet( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + type: 'table', + title: dupRelation.tn, + }, + ); + + const meta = JSON.parse(tnModel.meta); + + const duplicateBts = meta.belongsTo.filter( + (bt) => + bt.tn === dupRelation.tn && + bt.rtn === dupRelation.rtn && + bt.cn === dupRelation.cn && + bt.rcn === dupRelation.rcn && + bt.type === 'real', + ); + + if (duplicateBts?.length > 1) { + meta.belongsTo.splice(meta.belongsTo.indexOf(duplicateBts[1]), 1); + } + + await ctx.xcMeta.metaUpdate( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { meta: JSON.stringify(meta) }, + { + type: 'table', + title: dupRelation.tn, + }, + ); + } + { + const rtnModel = await ctx.xcMeta.metaGet( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { + type: 'table', + title: dupRelation.rtn, + }, + ); + + const meta = JSON.parse(rtnModel.meta); + + const duplicateHms = meta.hasMany.filter( + (bt) => + bt.tn === dupRelation.tn && + bt.rtn === dupRelation.rtn && + bt.cn === dupRelation.cn && + bt.rcn === dupRelation.rcn && + bt.type === 'real', + ); + + if (duplicateHms?.length > 1) { + meta.hasMany.splice(meta.hasMany.indexOf(duplicateHms[1]), 1); + } + await ctx.xcMeta.metaUpdate( + ctx.projectId, + ctx.dbAlias, + 'nc_models', + { meta: JSON.stringify(meta) }, + { + type: 'table', + title: dupRelation.rtn, + }, + ); + } + } + } catch (e) { + console.log(e); + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/rest/RestApiBuilder.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/rest/RestApiBuilder.ts new file mode 100644 index 0000000000..0c09051210 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/rest/RestApiBuilder.ts @@ -0,0 +1,178 @@ +import { Router } from 'express'; +import debug from 'debug'; +import autoBind from 'auto-bind'; +import SwaggerXc from '../../../db/sql-mgr/code/routers/xc-ts/SwaggerXc' +import ExpressXcTsRoutes from '../../../db/sql-mgr/code/routes/xc-ts/ExpressXcTsRoutes' +import { MetaService } from '../../../meta/meta.service'; +import Noco from '../../../Noco'; +import NcHelp from '../../../utils/NcHelp' +import type { DbConfig, NcConfig } from '../../../interface/config'; +import type NcProjectBuilder from '../NcProjectBuilder'; +import BaseApiBuilder, { XcTablesPopulateParams } from '../BaseApiBuilder'; + +const log = debug('nc:api:rest'); + +export class RestApiBuilder extends BaseApiBuilder { + public readonly type = 'rest'; + + + protected nocoTypes: any; + protected nocoRootResolvers: any; + private routers: { [key: string]: Router }; + + constructor( + app: Noco, + projectBuilder: NcProjectBuilder, + config: NcConfig, + connectionConfig: DbConfig, + xcMeta?: MetaService, + ) { + super(app, projectBuilder, config, connectionConfig); + autoBind(this); + this.routers = {}; + this.hooks = {}; + this.xcMeta = xcMeta; + } + + public async init(): Promise { + await super.init(); + } + + protected async ncUpManyToMany(ctx: any): Promise { + const metas = await super.ncUpManyToMany(ctx); + if (!metas) { + return; + } + for (const meta of metas) { + const ctx = this.generateContextForTable( + meta.tn, + meta.columns, + [], + meta.hasMany, + meta.belongsTo, + meta.type, + meta._tn + ); + + /* create routes for table */ + const routes = new ExpressXcTsRoutes({ + dir: '', + ctx, + filename: '', + }).getObjectWithoutFunctions(); + + /* create nc_routes, add new routes or update order */ + const routesInsertion = routes.map((route, i) => { + return async () => { + if ( + !(await this.xcMeta.metaGet( + this.projectId, + this.dbAlias, + 'nc_routes', + { + path: route.path, + tn: meta.tn, + title: meta.tn, + type: route.type, + } + )) + ) { + await this.xcMeta.metaInsert( + this.projectId, + this.dbAlias, + 'nc_routes', + { + acl: JSON.stringify(route.acl), + handler: JSON.stringify(route.handler), + order: i, + path: route.path, + tn: meta.tn, + title: meta.tn, + type: route.type, + } + ); + } else { + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_routes', + { + order: i, + }, + { + path: route.path, + tn: meta.tn, + title: meta.tn, + type: route.type, + } + ); + } + }; + }); + + await NcHelp.executeOperations( + routesInsertion, + this.connectionConfig.client + ); + } + + // add new routes + } + + protected async getManyToManyRelations(args = {}): Promise> { + const metas: Set = await super.getManyToManyRelations(args); + + for (const metaObj of metas) { + const ctx = this.generateContextForTable( + metaObj.tn, + metaObj.columns, + [...metaObj.belongsTo, ...metaObj.hasMany], + metaObj.hasMany, + metaObj.belongsTo + ); + + const swaggerDoc = await new SwaggerXc({ + dir: '', + ctx: { + ...ctx, + v: metaObj.v, + }, + filename: '', + }).getObject(); + + const meta = await this.xcMeta.metaGet( + this.projectId, + this.dbAlias, + 'nc_models', + { + title: metaObj.tn, + type: 'table', + } + ); + const oldSwaggerDoc = JSON.parse(meta.schema); + + // // keep upto 5 schema backup on table update + // let previousSchemas = [oldSwaggerDoc] + // if (meta.schema_previous) { + // previousSchemas = [...JSON.parse(meta.schema_previous), oldSwaggerDoc].slice(-5); + // } + + oldSwaggerDoc.definitions = swaggerDoc.definitions; + await this.xcMeta.metaUpdate( + this.projectId, + this.dbAlias, + 'nc_models', + { + schema: JSON.stringify(oldSwaggerDoc), + // schema_previous: JSON.stringify(previousSchemas) + }, + { + title: metaObj.tn, + type: 'table', + } + ); + } + + return metas; + } +} diff --git a/packages/nocodb-nest/src/version-upgrader/v1-legacy/templates/NcTemplateParser.ts b/packages/nocodb-nest/src/version-upgrader/v1-legacy/templates/NcTemplateParser.ts new file mode 100644 index 0000000000..8a7a011d74 --- /dev/null +++ b/packages/nocodb-nest/src/version-upgrader/v1-legacy/templates/NcTemplateParser.ts @@ -0,0 +1,253 @@ +import { SqlUiFactory, UITypes } from 'nocodb-sdk'; +import type { MssqlUi, MysqlUi, OracleUi, PgUi, SqliteUi } from 'nocodb-sdk'; + +export default class NcTemplateParser { + sqlUi: + | typeof MysqlUi + | typeof MssqlUi + | typeof PgUi + | typeof OracleUi + | typeof SqliteUi; + + private _tables: any[]; + private client: string; + private _relations: any[]; + private _m2mRelations: any[]; + private _virtualColumns: { [tn: string]: any[] }; + private prefix: string; + private template: any; + + constructor({ client, template, prefix = '' }) { + this.client = client; + this.sqlUi = SqlUiFactory.create({ client }); + this.template = template; + this.prefix = prefix; + } + + public parse(template?: any): any { + const tables = []; + this.template = template || this.template; + const tableTemplates = this.template.tables.map((tableTemplate) => { + const t = { + ...tableTemplate, + tn: this.getTable(tableTemplate.tn), + _tn: tableTemplate._tn || tableTemplate.tn, + }; + const table = this.extractTable(t); + tables.push(table); + return t; + }); + + this._tables = tables; + + for (const tableTemplate of tableTemplates) { + this.extractRelations(tableTemplate); + this.extractVirtualColumns(tableTemplate); + } + } + + private extractTable(tableTemplate) { + if (!tableTemplate?.tn) { + throw Error('Missing table name in template'); + } + + const defaultColumns = this.sqlUi + .getNewTableColumns() + .filter( + (column) => + column.cn !== 'title' && + (column.uidt !== 'ID' || + tableTemplate.columns.every((c) => c.uidt !== 'ID')) + ); + + return { + tn: tableTemplate.tn, + _tn: tableTemplate._tn, + columns: [ + defaultColumns[0], + ...this.extractTableColumns(tableTemplate.columns), + ...defaultColumns.slice(1), + ], + }; + } + + private extractTableColumns(tableColumns: any[]) { + const columns = []; + for (const tableColumn of tableColumns) { + if (!tableColumn?.cn) { + throw Error('Missing column name in template'); + } + switch (tableColumn.uidt) { + // case UITypes.ForeignKey: + // // todo : + // this.extractRelations(tableColumn, 'bt'); + // break; + // case UITypes.LinkToAnotherRecord: + // // todo : + // this.extractRelations(tableColumn, 'hm'); + // // this.extractRelations(tableColumn, 'mm'); + // break; + default: + { + const colProp = this.sqlUi.getDataTypeForUiType(tableColumn); + columns.push({ + ...this.sqlUi.getNewColumn(''), + rqd: false, + pk: false, + ai: false, + cdf: null, + un: false, + dtx: 'specificType', + dtxp: this.sqlUi.getDefaultLengthForDatatype(colProp.dt), + dtxs: this.sqlUi.getDefaultScaleForDatatype(colProp.dt), + ...colProp, + _cn: tableColumn.cn, + ...tableColumn, + }); + } + break; + } + } + return columns; + } + + protected extractRelations(tableTemplate) { + if (!this._relations) this._relations = []; + if (!this._m2mRelations) this._m2mRelations = []; + for (const hasMany of tableTemplate.hasMany || []) { + const childTable = this.tables.find( + (table) => table.tn === this.getTable(hasMany.tn) + ); + const parentTable = this.tables.find( + (table) => table.tn === tableTemplate.tn + ); + const parentPrimaryColumn = parentTable.columns.find( + (column) => column.uidt === UITypes.ID + ); + // + // // if duplicate relation ignore + // if ( + // this._relations.some(rl => { + // return ( + // (rl.childTable === childTable.tn && + // rl.parentTable === parentTable.tn) || + // (rl.parentTable === childTable.tn && + // rl.childTable === parentTable.tn) + // ); + // }) + // ) { + // continue;f + // } + + // add a column in child table + const childColumnName = `${tableTemplate.tn}_id`; + + childTable.columns.push({ + column_name: childColumnName, + _cn: childColumnName, + rqd: false, + pk: false, + ai: false, + cdf: null, + dt: parentPrimaryColumn.dt, + dtxp: parentPrimaryColumn.dtxp, + dtxs: parentPrimaryColumn.dtxs, + un: parentPrimaryColumn.un, + altered: 1, + }); + + // add relation create entry + this._relations.push({ + childColumn: childColumnName, + childTable: childTable.tn, + onDelete: 'NO ACTION', + onUpdate: 'NO ACTION', + parentColumn: parentPrimaryColumn.cn, + parentTable: tableTemplate.tn, + type: this.client === 'sqlite3' ? 'virtual' : 'real', + updateRelation: false, + }); + } + for (const manyToMany of tableTemplate.manyToMany || []) { + // @ts-ignore + const childTable = this.tables.find( + (table) => table.tn === this.getTable(manyToMany.rtn) + ); + const parentTable = this.tables.find( + (table) => table.tn === tableTemplate.tn + ); + const parentPrimaryColumn = parentTable.columns.find( + (column) => column.uidt === UITypes.ID + ); + const childPrimaryColumn = childTable.columns.find( + (column) => column.uidt === UITypes.ID + ); + + // if duplicate relation ignore + if ( + this._m2mRelations.some((mm) => { + return ( + (mm.childTable === childTable.tn && + mm.parentTable === parentTable.tn) || + (mm.parentTable === childTable.tn && + mm.childTable === parentTable.tn) + ); + }) + ) { + continue; + } + + // add many to many relation create entry + this._m2mRelations.push({ + alias: 'title8', + childColumn: childPrimaryColumn.cn, + childTable: childTable.tn, + onDelete: 'NO ACTION', + onUpdate: 'NO ACTION', + parentColumn: parentPrimaryColumn.cn, + parentTable: parentTable.tn, + type: this.client === 'sqlite3' ? 'virtual' : 'real', + updateRelation: false, + }); + } + } + + private extractVirtualColumns(tableMeta) { + if (!this._virtualColumns) this._virtualColumns = {}; + const virtualColumns = []; + for (const v of tableMeta.v || []) { + const v1 = { ...v }; + + if (v.rl) { + v1.rl.rlttn = v1.rl.rltn; + v1.rl.rltn = this.getTable(v1.rl.rltn); + } else if (v.lk) { + v1.lk._ltn = v1.lk.ltn; + v1.lk.ltn = this.getTable(v1.lk.ltn); + } + + virtualColumns.push(v1); + } + this.virtualColumns[tableMeta.tn] = virtualColumns; + } + + get tables(): any[] { + return this._tables; + } + + get relations(): any[] { + return this._relations; + } + + get m2mRelations(): any[] { + return this._m2mRelations; + } + + get virtualColumns(): { [tn: string]: any[] } { + return this._virtualColumns; + } + + private getTable(tn) { + return `${this.prefix}${tn}`; + } +}