feat: introduce app upgrader

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb-nest/src/app.module.ts

@@ -51,6 +51,7 @@ import type {
   MiddlewareConsumer,
   OnApplicationBootstrap,
 } from '@nestjs/common';
+import NcUpgrader from './version-upgrader/NcUpgrader';
 
 @Module({
   imports: [
@@ -130,5 +131,8 @@ export class AppModule implements OnApplicationBootstrap {
     // temporary hack
     Noco._ncMeta = this.metaService;
     Noco.config = this.connection.config;
+
+    // run upgrader
+    await NcUpgrader.upgrade({ ncMeta: Noco._ncMeta });
   }
 }

packages/nocodb-nest/src/interface/XcDynamicChanges.ts

@@ -1,8 +0,0 @@
-export default interface XcDynamicChanges {
-  onTableCreate(tn: string): Promise<void>;
-  onTableUpdate(changeObj: any): Promise<void>;
-  onTableDelete(tn: string): Promise<void>;
-  onTableRename(oldTableName: string, newTableName: string): Promise<void>;
-  onHandlerCodeUpdate(tn: string): Promise<void>;
-  onMetaUpdate(tn: string): Promise<void>;
-}

packages/nocodb-nest/src/version-upgrader/NcUpgrader.ts

@@ -0,0 +1,151 @@
+import debug from 'debug';
+import { T } from 'nc-help';
+import boxen from 'boxen';
+import { NcConfig } from '../interface/config';
+import { MetaService } from '../meta/meta.service';
+import ncAttachmentUpgrader from './ncAttachmentUpgrader';
+import ncAttachmentUpgrader_0104002 from './ncAttachmentUpgrader_0104002';
+import ncStickyColumnUpgrader from './ncStickyColumnUpgrader';
+import ncFilterUpgrader_0104004 from './ncFilterUpgrader_0104004';
+import ncFilterUpgrader_0105003 from './ncFilterUpgrader_0105003';
+import ncFilterUpgrader from './ncFilterUpgrader';
+import ncProjectRolesUpgrader from './ncProjectRolesUpgrader';
+import ncDataTypesUpgrader from './ncDataTypesUpgrader';
+import ncProjectUpgraderV2_0090000 from './ncProjectUpgraderV2_0090000';
+import ncProjectEnvUpgrader0011045 from './ncProjectEnvUpgrader0011045';
+import ncProjectEnvUpgrader from './ncProjectEnvUpgrader';
+import ncHookUpgrader from './ncHookUpgrader';
+
+const log = debug('nc:version-upgrader');
+
+export interface NcUpgraderCtx {
+  ncMeta: MetaService;
+}
+
+export default class NcUpgrader {
+  private static STORE_KEY = 'NC_CONFIG_MAIN';
+
+  // Todo: transaction
+  public static async upgrade(ctx: NcUpgraderCtx): Promise<any> {
+    this.log(`upgrade :`);
+    let oldVersion;
+
+    try {
+      ctx.ncMeta = await ctx.ncMeta.startTransaction();
+
+      const NC_VERSIONS: any[] = [
+        { name: '0009000', handler: null },
+        { name: '0009044', handler: null },
+        { name: '0011043', handler: ncProjectEnvUpgrader },
+        { name: '0011045', handler: ncProjectEnvUpgrader0011045 },
+        { name: '0090000', handler: ncProjectUpgraderV2_0090000 },
+        { name: '0098004', handler: ncDataTypesUpgrader },
+        { name: '0098005', handler: ncProjectRolesUpgrader },
+        { name: '0100002', handler: ncFilterUpgrader },
+        { name: '0101002', handler: ncAttachmentUpgrader },
+        { name: '0104002', handler: ncAttachmentUpgrader_0104002 },
+        { name: '0104004', handler: ncFilterUpgrader_0104004 },
+        { name: '0105002', handler: ncStickyColumnUpgrader },
+        { name: '0105003', handler: ncFilterUpgrader_0105003 },
+        { name: '0105004', handler: ncHookUpgrader },
+      ];
+      if (!(await ctx.ncMeta.knexConnection?.schema?.hasTable?.('nc_store'))) {
+        return;
+      }
+      this.log(`upgrade : Getting configuration from meta database`);
+
+      const config = await ctx.ncMeta.metaGet('', '', 'nc_store', {
+        key: this.STORE_KEY,
+      });
+
+      if (config) {
+        const configObj: NcConfig = JSON.parse(config.value);
+        if (configObj.version !== process.env.NC_VERSION) {
+          oldVersion = configObj.version;
+          for (const version of NC_VERSIONS) {
+            // compare current version and old version
+            if (version.name > configObj.version) {
+              this.log(
+                `upgrade : Upgrading '%s' => '%s'`,
+                configObj.version,
+                version.name,
+              );
+              await version?.handler?.(ctx);
+
+              // update version in meta after each upgrade
+              config.version = version.name;
+              await ctx.ncMeta.metaUpdate(
+                '',
+                '',
+                'nc_store',
+                {
+                  value: JSON.stringify(config),
+                },
+                {
+                  key: NcUpgrader.STORE_KEY,
+                },
+              );
+
+              // todo: backup data
+            }
+            if (version.name === process.env.NC_VERSION) {
+              break;
+            }
+          }
+          config.version = process.env.NC_VERSION;
+        }
+      } else {
+        this.log(`upgrade : Inserting config to meta database`);
+        const configObj: any = {};
+        const isOld = (await ctx.ncMeta.projectList())?.length;
+        configObj.version = isOld ? '0009000' : process.env.NC_VERSION;
+        await ctx.ncMeta.metaInsert('', '', 'nc_store', {
+          key: NcUpgrader.STORE_KEY,
+          value: JSON.stringify(configObj),
+        });
+        if (isOld) {
+          await this.upgrade(ctx);
+        }
+      }
+      await ctx.ncMeta.commit();
+      T.emit('evt', {
+        evt_type: 'appMigration:upgraded',
+        from: oldVersion,
+        to: process.env.NC_VERSION,
+      });
+    } catch (e) {
+      await ctx.ncMeta.rollback(e);
+      T.emit('evt', {
+        evt_type: 'appMigration:failed',
+        from: oldVersion,
+        to: process.env.NC_VERSION,
+        msg: e.message,
+        err: e?.stack?.split?.('\n').slice(0, 2).join('\n'),
+      });
+      console.log(getUpgradeErrorLog(e, oldVersion, process.env.NC_VERSION));
+      throw e;
+    }
+  }
+
+  private static log(str, ...args): void {
+    log(`${str}`, ...args);
+  }
+}
+
+function getUpgradeErrorLog(e: Error, oldVersion: string, newVersion: string) {
+  const errorTitle = `Migration from ${oldVersion} to ${newVersion} failed`;
+
+  return boxen(
+    `Error
+-----
+${e.stack}
+
+
+Please raise an issue in our github by using following link : 
+https://github.com/nocodb/nocodb/issues/new?labels=Type%3A%20Bug&template=bug_report.md
+
+Or contact us in our Discord community by following link :
+https://discord.gg/5RgZmkW ( message @o1lab, @pranavxc or @wingkwong )`,
+    { title: errorTitle, padding: 1, borderColor: 'yellow' },
+  );
+}

packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader.ts

@@ -0,0 +1,190 @@
+import { UITypes } from 'nocodb-sdk';
+import { XKnex } from '../db/CustomKnex';
+import { MetaTable } from '../utils/globals';
+import Base from '../models/Base';
+import Model from '../models/Model';
+import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
+import { throwTimeoutError } from './ncUpgradeErrors';
+import type { Knex } from 'knex';
+import type { NcUpgraderCtx } from './NcUpgrader';
+// import type { XKnex } from '../db/sql-data-mapper';
+import type { BaseType } from 'nocodb-sdk';
+
+// before 0.103.0, an attachment object was like
+// [{
+//   "url": "http://localhost:8080/download/noco/xcdb/Sheet-1/title5/39A410.jpeg",
+//   "title": "foo.jpeg",
+//   "mimetype": "image/jpeg",
+//   "size": 6494
+// }]
+// in this way, if the base url is changed, the url will be broken
+// this upgrader is to convert the existing local attachment object to the following format
+// [{
+//   "url": "http://localhost:8080/download/noco/xcdb/Sheet-1/title5/39A410.jpeg",
+//   "path": "download/noco/xcdb/Sheet-1/title5/39A410.jpeg",
+//   "title": "foo.jpeg",
+//   "mimetype": "image/jpeg",
+//   "size": 6494
+// }]
+// the new url will be constructed by `${ncSiteUrl}/${path}` in UI. the old url will be used for fallback
+// while other non-local attachments will remain unchanged
+
+function getTnPath(knex: XKnex, tb: Model) {
+  const schema = (knex as any).searchPath?.();
+  const clientType = knex.clientType();
+  if (clientType === 'mssql' && schema) {
+    return knex.raw('??.??', [schema, tb.table_name]).toQuery();
+  } else if (clientType === 'snowflake') {
+    return [
+      knex.client.config.connection.database,
+      knex.client.config.connection.schema,
+      tb.table_name,
+    ].join('.');
+  } else {
+    return tb.table_name;
+  }
+}
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const bases: BaseType[] = await ncMeta.metaList2(null, null, MetaTable.BASES);
+  for (const _base of bases) {
+    const base = new Base(_base);
+
+    // skip if the project_id is missing
+    if (!base.project_id) {
+      continue;
+    }
+
+    const project = await ncMeta.metaGet2(null, null, MetaTable.PROJECT, {
+      id: base.project_id,
+    });
+
+    // skip if the project is missing
+    if (!project) {
+      continue;
+    }
+
+    const isProjectDeleted = project.deleted;
+
+    const knex: Knex = base.is_meta
+      ? ncMeta.knexConnection
+      : await NcConnectionMgrv2.get(base);
+    const models = await base.getModels(ncMeta);
+
+    // used in timeout error message
+    const timeoutErrorInfo = {
+      projectTitle: project.title,
+      connection: knex.client.config.connection,
+    };
+
+    for (const model of models) {
+      try {
+        // if the table is missing in database, skip
+        if (!(await knex.schema.hasTable(getTnPath(knex, model)))) {
+          continue;
+        }
+
+        const updateRecords = [];
+
+        // get all attachment & primary key columns
+        // and filter out the columns that are missing in database
+        const columns = await (await Model.get(model.id, ncMeta))
+          .getColumns(ncMeta)
+          .then(async (columns) => {
+            const filteredColumns = [];
+
+            for (const column of columns) {
+              if (column.uidt !== UITypes.Attachment && !column.pk) continue;
+              if (
+                !(await knex.schema.hasColumn(
+                  getTnPath(knex, model),
+                  column.column_name,
+                ))
+              )
+                continue;
+              filteredColumns.push(column);
+            }
+
+            return filteredColumns;
+          });
+
+        const attachmentColumns = columns
+          .filter((c) => c.uidt === UITypes.Attachment)
+          .map((c) => c.column_name);
+        if (attachmentColumns.length === 0) {
+          continue;
+        }
+        const primaryKeys = columns
+          .filter((c) => c.pk)
+          .map((c) => c.column_name);
+
+        const records = await knex(getTnPath(knex, model)).select();
+
+        for (const record of records) {
+          for (const attachmentColumn of attachmentColumns) {
+            let attachmentMeta: Array<{
+              url: string;
+            }>;
+
+            // if parsing failed ignore the cell
+            try {
+              attachmentMeta =
+                typeof record[attachmentColumn] === 'string'
+                  ? JSON.parse(record[attachmentColumn])
+                  : record[attachmentColumn];
+            } catch {}
+
+            // if cell data is not an array, ignore it
+            if (!Array.isArray(attachmentMeta)) {
+              continue;
+            }
+
+            if (attachmentMeta) {
+              const newAttachmentMeta = [];
+              for (const attachment of attachmentMeta) {
+                if ('url' in attachment && typeof attachment.url === 'string') {
+                  const match = attachment.url.match(/^(.*)\/download\/(.*)$/);
+                  if (match) {
+                    // e.g. http://localhost:8080/download/noco/xcdb/Sheet-1/title5/ee2G8p_nute_gunray.png
+                    // match[1] = http://localhost:8080
+                    // match[2] = download/noco/xcdb/Sheet-1/title5/ee2G8p_nute_gunray.png
+                    const path = `download/${match[2]}`;
+
+                    newAttachmentMeta.push({
+                      ...attachment,
+                      path,
+                    });
+                  } else {
+                    // keep it as it is
+                    newAttachmentMeta.push(attachment);
+                  }
+                }
+              }
+              const where = primaryKeys
+                .map((key) => {
+                  return { [key]: record[key] };
+                })
+                .reduce((acc, val) => Object.assign(acc, val), {});
+              updateRecords.push(
+                await knex(getTnPath(knex, model))
+                  .update({
+                    [attachmentColumn]: JSON.stringify(newAttachmentMeta),
+                  })
+                  .where(where),
+              );
+            }
+          }
+        }
+        await Promise.all(updateRecords);
+      } catch (e) {
+        // ignore the error related to deleted project
+        if (!isProjectDeleted) {
+          // throw the custom timeout error message if applicable
+          throwTimeoutError(e, timeoutErrorInfo);
+          // throw general error
+          throw e;
+        }
+      }
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncAttachmentUpgrader_0104002.ts

@@ -0,0 +1,170 @@
+import { UITypes } from 'nocodb-sdk';
+import { XKnex } from '../db/CustomKnex'
+import { MetaTable } from '../utils/globals';
+import Base from '../models/Base';
+import Model from '../models/Model';
+import NcConnectionMgrv2 from '../utils/common/NcConnectionMgrv2';
+import { throwTimeoutError } from './ncUpgradeErrors';
+import type { Knex } from 'knex';
+import type { NcUpgraderCtx } from './NcUpgrader';
+import type { BaseType } from 'nocodb-sdk';
+
+// after 0101002 upgrader, the attachment object would become broken when
+// (1) switching views after updating a singleSelect field
+// since `url` will be enriched the attachment cell, and `saveOrUpdateRecords` in Grid.vue will be triggered
+// in this way, the attachment value will be corrupted like
+// {"{\"path\":\"download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg\",\"title\":\"haha.jpeg\",\"mimetype\":\"image/jpeg\",\"size\":6494,\"url\":\"http://localhost:8080/download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg\"}"}
+// while the expected one is
+// [{"path":"download/noco/xcdb2/attachment2/a/JRubxMQgPlcumdm3jL.jpeg","title":"haha.jpeg","mimetype":"image/jpeg","size":6494}]
+// (2) or reordering attachments
+// since the incoming value is not string, the value will be broken
+// hence, this upgrader is to revert back these corrupted values
+
+function getTnPath(knex: XKnex, tb: Model) {
+  const schema = (knex as any).searchPath?.();
+  const clientType = knex.clientType();
+  if (clientType === 'mssql' && schema) {
+    return knex.raw('??.??', [schema, tb.table_name]).toQuery();
+  } else if (clientType === 'snowflake') {
+    return [
+      knex.client.config.connection.database,
+      knex.client.config.connection.schema,
+      tb.table_name,
+    ].join('.');
+  } else {
+    return tb.table_name;
+  }
+}
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const bases: BaseType[] = await ncMeta.metaList2(null, null, MetaTable.BASES);
+  for (const _base of bases) {
+    const base = new Base(_base);
+
+    // skip if the project_id is missing
+    if (!base.project_id) {
+      continue;
+    }
+
+    const project = await ncMeta.metaGet2(null, null, MetaTable.PROJECT, {
+      id: base.project_id,
+    });
+
+    // skip if the project is missing
+    if (!project) {
+      continue;
+    }
+
+    const isProjectDeleted = project.deleted;
+
+    const knex: Knex = base.is_meta
+      ? ncMeta.knexConnection
+      : await NcConnectionMgrv2.get(base);
+    const models = await base.getModels(ncMeta);
+
+    // used in timeout error message
+    const timeoutErrorInfo = {
+      projectTitle: project.title,
+      connection: knex.client.config.connection,
+    };
+
+    for (const model of models) {
+      try {
+        // if the table is missing in database, skip
+        if (!(await knex.schema.hasTable(getTnPath(knex, model)))) {
+          continue;
+        }
+
+        const updateRecords = [];
+
+        // get all attachment & primary key columns
+        // and filter out the columns that are missing in database
+        const columns = await (await Model.get(model.id, ncMeta))
+          .getColumns(ncMeta)
+          .then(async (columns) => {
+            const filteredColumns = [];
+
+            for (const column of columns) {
+              if (column.uidt !== UITypes.Attachment && !column.pk) continue;
+              if (
+                !(await knex.schema.hasColumn(
+                  getTnPath(knex, model),
+                  column.column_name
+                ))
+              )
+                continue;
+              filteredColumns.push(column);
+            }
+
+            return filteredColumns;
+          });
+
+        const attachmentColumns = columns
+          .filter((c) => c.uidt === UITypes.Attachment)
+          .map((c) => c.column_name);
+        if (attachmentColumns.length === 0) {
+          continue;
+        }
+        const primaryKeys = columns
+          .filter((c) => c.pk)
+          .map((c) => c.column_name);
+
+        const records = await knex(getTnPath(knex, model)).select();
+
+        for (const record of records) {
+          const where = primaryKeys
+            .map((key) => {
+              return { [key]: record[key] };
+            })
+            .reduce((acc, val) => Object.assign(acc, val), {});
+          for (const attachmentColumn of attachmentColumns) {
+            if (typeof record[attachmentColumn] === 'string') {
+              // potentially corrupted
+              try {
+                JSON.parse(record[attachmentColumn]);
+                // it works fine - skip
+                continue;
+              } catch {
+                try {
+                  // corrupted
+                  let corruptedAttachment = record[attachmentColumn];
+                  // replace the first and last character with `[` and `]`
+                  // and parse it again
+                  corruptedAttachment = JSON.parse(
+                    `[${corruptedAttachment.slice(
+                      1,
+                      corruptedAttachment.length - 1
+                    )}]`
+                  );
+                  const newAttachmentMeta = [];
+                  for (const attachment of corruptedAttachment) {
+                    newAttachmentMeta.push(JSON.parse(attachment));
+                  }
+                  updateRecords.push(
+                    await knex(getTnPath(knex, model))
+                      .update({
+                        [attachmentColumn]: JSON.stringify(newAttachmentMeta),
+                      })
+                      .where(where)
+                  );
+                } catch {
+                  // if parsing failed ignore the cell
+                  continue;
+                }
+              }
+            }
+          }
+        }
+        await Promise.all(updateRecords);
+      } catch (e) {
+        // ignore the error related to deleted project
+        if (!isProjectDeleted) {
+          // throw the custom timeout error message if applicable
+          throwTimeoutError(e, timeoutErrorInfo);
+          // throw general error
+          throw e;
+        }
+      }
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncDataTypesUpgrader.ts

@@ -0,0 +1,14 @@
+import { UITypes } from 'nocodb-sdk';
+import { MetaTable } from '../utils/globals';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+// The Count and AutoNumber types are removed
+// so convert all existing Count and AutoNumber fields to Number type
+export default async function (ctx: NcUpgraderCtx) {
+  // directly update uidt of all existing Count and AutoNumber fields to Number
+  await ctx.ncMeta.knex
+    .update({ uidt: UITypes.Number })
+    .where({ uidt: UITypes.Count })
+    .orWhere({ uidt: UITypes.AutoNumber })
+    .table(MetaTable.COLUMNS);
+}

packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader.ts

@@ -0,0 +1,40 @@
+import { MetaTable } from '../utils/globals';
+import View from '../models/View';
+import Hook from '../models/Hook';
+import Column from '../models/Column';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+// before 0.101.0, an incorrect project_id was inserted when
+// a filter is created without specifying the column
+// this upgrader is to retrieve the correct project id from either view, hook, or column
+// and update the project id
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
+  for (const filter of filters) {
+    let model: { project_id?: string; base_id?: string };
+    if (filter.fk_view_id) {
+      model = await View.get(filter.fk_view_id, ncMeta);
+    } else if (filter.fk_hook_id) {
+      model = await Hook.get(filter.fk_hook_id, ncMeta);
+    } else if (filter.fk_column_id) {
+      model = await Column.get({ colId: filter.fk_column_id }, ncMeta);
+    } else {
+      continue;
+    }
+
+    // skip if related model is not found
+    if (!model) {
+      continue;
+    }
+
+    if (filter.project_id !== model.project_id) {
+      await ncMeta.metaUpdate(
+        null,
+        null,
+        MetaTable.FILTER_EXP,
+        { base_id: model.base_id, project_id: model.project_id },
+        filter.id
+      );
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0104004.ts

@@ -0,0 +1,359 @@
+import { UITypes } from 'nocodb-sdk';
+import { MetaService } from '../meta/meta.service'
+import { MetaTable } from '../utils/globals';
+import Column from '../models/Column';
+import Filter from '../models/Filter';
+import Project from '../models/Project';
+import type { NcUpgraderCtx } from './NcUpgrader';
+import type { SelectOptionsType } from 'nocodb-sdk';
+
+// as of 0.104.3, almost all filter operators are available to all column types
+// while some of them aren't supposed to be shown
+// this upgrader is to remove those unsupported filters / migrate to the correct filter
+
+// Change Summary:
+// - Text-based columns:
+//     - remove `>`, `<`, `>=`, `<=`
+//     - Numeric-based / SingleSelect columns:
+//     - remove `like`
+//     - migrate `null`, and `empty` to `blank`
+// - Checkbox columns:
+//     - remove `equal`
+//     - migrate `empty` and `null` to `notchecked`
+// - MultiSelect columns:
+//     - remove `like`
+//     - migrate `equal`, `null`, `empty`
+//     - Attachment columns:
+//     - remove `>`, `<`, `>=`, `<=`, `equal`
+//     - migrate `empty`, `null` to `blank`
+// - LTAR columns:
+//     - remove `>`, `<`, `>=`, `<=`
+//     - migrate `empty`, `null` to `blank`
+// - Lookup columns:
+//     - migrate `empty`, `null` to `blank`
+// - Duration columns:
+//     - remove  `like`
+//     - migrate `empty`, `null` to `blank`
+
+const removeEqualFilters = (filter, ncMeta) => {
+  const actions = [];
+  // remove `is equal`, `is not equal`
+  if (['eq', 'neq'].includes(filter.comparison_op)) {
+    actions.push(Filter.delete(filter.id, ncMeta));
+  }
+  return actions;
+};
+
+const removeArithmeticFilters = (filter, ncMeta) => {
+  const actions = [];
+  // remove `>`, `<`, `>=`, `<=`
+  if (['gt', 'lt', 'gte', 'lte'].includes(filter.comparison_op)) {
+    actions.push(Filter.delete(filter.id, ncMeta));
+  }
+  return actions;
+};
+
+const removeLikeFilters = (filter, ncMeta) => {
+  const actions = [];
+  // remove `is like`, `is not like`
+  if (['like', 'nlike'].includes(filter.comparison_op)) {
+    actions.push(Filter.delete(filter.id, ncMeta));
+  }
+  return actions;
+};
+
+const migrateNullAndEmptyToBlankFilters = (filter, ncMeta) => {
+  const actions = [];
+  if (['empty', 'null'].includes(filter.comparison_op)) {
+    // migrate to blank
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'blank',
+        },
+        ncMeta
+      )
+    );
+  } else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
+    // migrate to not blank
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'notblank',
+        },
+        ncMeta
+      )
+    );
+  }
+  return actions;
+};
+
+const migrateMultiSelectEq = async (filter, col: Column, ncMeta) => {
+  // only allow eq / neq
+  if (!['eq', 'neq'].includes(filter.comparison_op)) return;
+  // if there is no value -> delete this filter
+  if (!filter.value) {
+    return await Filter.delete(filter.id, ncMeta);
+  }
+  // options inputted from users
+  const options = filter.value.split(',');
+  // retrieve the possible col options
+  const colOptions = (await col.getColOptions()) as SelectOptionsType;
+  // only include valid options as the input value becomes dropdown type now
+  const validOptions = [];
+  for (const option of options) {
+    if (colOptions.options.includes(option)) {
+      validOptions.push(option);
+    }
+  }
+  const newFilterValue = validOptions.join(',');
+  // if all inputted options are invalid -> delete this filter
+  if (!newFilterValue) {
+    return await Filter.delete(filter.id, ncMeta);
+  }
+  const actions = [];
+  if (filter.comparison_op === 'eq') {
+    // migrate to `contains all of`
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'anyof',
+          value: newFilterValue,
+        },
+        ncMeta
+      )
+    );
+  } else if (filter.comparison_op === 'neq') {
+    // migrate to `doesn't contain all of`
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'nanyof',
+          value: newFilterValue,
+        },
+        ncMeta
+      )
+    );
+  }
+  return await Promise.all(actions);
+};
+
+const migrateToCheckboxFilter = (filter, ncMeta) => {
+  const actions = [];
+  const possibleTrueValues = ['true', 'True', '1', 'T', 'Y'];
+  const possibleFalseValues = ['false', 'False', '0', 'F', 'N'];
+  if (['empty', 'null'].includes(filter.comparison_op)) {
+    // migrate to not checked
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'notchecked',
+        },
+        ncMeta
+      )
+    );
+  } else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
+    //  migrate to checked
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'checked',
+        },
+        ncMeta
+      )
+    );
+  } else if (filter.comparison_op === 'eq') {
+    if (possibleTrueValues.includes(filter.value)) {
+      // migrate to checked
+      actions.push(
+        Filter.update(
+          filter.id,
+          {
+            comparison_op: 'checked',
+            value: '',
+          },
+          ncMeta
+        )
+      );
+    } else if (possibleFalseValues.includes(filter.value)) {
+      // migrate to notchecked
+      actions.push(
+        Filter.update(
+          filter.id,
+          {
+            comparison_op: 'notchecked',
+            value: '',
+          },
+          ncMeta
+        )
+      );
+    } else {
+      // invalid value - good to delete
+      actions.push(Filter.delete(filter.id, ncMeta));
+    }
+  } else if (filter.comparison_op === 'neq') {
+    if (possibleFalseValues.includes(filter.value)) {
+      // migrate to checked
+      actions.push(
+        Filter.update(
+          filter.id,
+          {
+            comparison_op: 'checked',
+            value: '',
+          },
+          ncMeta
+        )
+      );
+    } else if (possibleTrueValues.includes(filter.value)) {
+      // migrate to not checked
+      actions.push(
+        Filter.update(
+          filter.id,
+          {
+            comparison_op: 'notchecked',
+            value: '',
+          },
+          ncMeta
+        )
+      );
+    } else {
+      // invalid value - good to delete
+      actions.push(Filter.delete(filter.id, ncMeta));
+    }
+  }
+  return actions;
+};
+
+async function migrateFilters(ncMeta: MetaService) {
+  const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
+  for (const filter of filters) {
+    if (!filter.fk_column_id || filter.is_group) {
+      continue;
+    }
+    const col = await Column.get({ colId: filter.fk_column_id }, ncMeta);
+    if (
+      [
+        UITypes.SingleLineText,
+        UITypes.LongText,
+        UITypes.PhoneNumber,
+        UITypes.Email,
+        UITypes.URL,
+      ].includes(col.uidt)
+    ) {
+      await Promise.all(removeArithmeticFilters(filter, ncMeta));
+    } else if (
+      [
+        // numeric fields
+        UITypes.Duration,
+        UITypes.Currency,
+        UITypes.Percent,
+        UITypes.Number,
+        UITypes.Decimal,
+        UITypes.Rating,
+        UITypes.Rollup,
+        // select fields
+        UITypes.SingleSelect,
+      ].includes(col.uidt)
+    ) {
+      await Promise.all([
+        ...removeLikeFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+    } else if (col.uidt === UITypes.Checkbox) {
+      await Promise.all(migrateToCheckboxFilter(filter, ncMeta));
+    } else if (col.uidt === UITypes.MultiSelect) {
+      await Promise.all([
+        ...removeLikeFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+      await migrateMultiSelectEq(filter, col, ncMeta);
+    } else if (col.uidt === UITypes.Attachment) {
+      await Promise.all([
+        ...removeArithmeticFilters(filter, ncMeta),
+        ...removeEqualFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+    } else if (col.uidt === UITypes.LinkToAnotherRecord) {
+      await Promise.all([
+        ...removeArithmeticFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+    } else if (col.uidt === UITypes.Lookup) {
+      await Promise.all([
+        ...removeArithmeticFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+    } else if (col.uidt === UITypes.Duration) {
+      await Promise.all([
+        ...removeLikeFilters(filter, ncMeta),
+        ...migrateNullAndEmptyToBlankFilters(filter, ncMeta),
+      ]);
+    }
+  }
+}
+
+async function updateProjectMeta(ncMeta: MetaService) {
+  const projectHasEmptyOrFilters: Record<string, boolean> = {};
+
+  const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
+
+  const actions = [];
+
+  for (const filter of filters) {
+    if (
+      ['notempty', 'notnull', 'empty', 'null'].includes(filter.comparison_op)
+    ) {
+      projectHasEmptyOrFilters[filter.project_id] = true;
+    }
+  }
+
+  const projects = await ncMeta.metaList2(null, null, MetaTable.PROJECT);
+
+  const defaultProjectMeta = {
+    showNullAndEmptyInFilter: false,
+  };
+
+  for (const project of projects) {
+    const oldProjectMeta = project.meta;
+    let newProjectMeta = defaultProjectMeta;
+    try {
+      newProjectMeta =
+        (typeof oldProjectMeta === 'string'
+          ? JSON.parse(oldProjectMeta)
+          : oldProjectMeta) ?? defaultProjectMeta;
+    } catch {}
+
+    newProjectMeta = {
+      ...newProjectMeta,
+      showNullAndEmptyInFilter: projectHasEmptyOrFilters[project.id] ?? false,
+    };
+
+    actions.push(
+      Project.update(
+        project.id,
+        {
+          meta: JSON.stringify(newProjectMeta),
+        },
+        ncMeta
+      )
+    );
+  }
+  await Promise.all(actions);
+}
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  // fix the existing filter behaviours or
+  // migrate `null` or `empty` filters to `blank`
+  await migrateFilters(ncMeta);
+  // enrich `showNullAndEmptyInFilter` in project meta
+  // if there is empty / null filters in existing projects,
+  // then set `showNullAndEmptyInFilter` to true
+  // else set to false
+  await updateProjectMeta(ncMeta);
+}

packages/nocodb-nest/src/version-upgrader/ncFilterUpgrader_0105003.ts

@@ -0,0 +1,101 @@
+import { UITypes } from 'nocodb-sdk';
+import { MetaService } from '../meta/meta.service';
+import { MetaTable } from '../utils/globals';
+import Column from '../models/Column';
+import Filter from '../models/Filter';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+// as of 0.105.3, year, time, date and datetime filters include `is like` and `is not like` which are not practical
+// `removeLikeAndNlikeFilters` in this upgrader is simply to remove them
+
+// besides, `null` and `empty` will be migrated to `blank` in `migrateEmptyAndNullFilters`
+
+// since the upcoming version will introduce a set of new filters for date / datetime with a new `comparison_sub_op`
+// `eq` and `neq` would become `is` / `is not` (comparison_op) + `exact date` (comparison_sub_op)
+// `migrateEqAndNeqFilters` in this upgrader is to add `exact date` in comparison_sub_op
+
+// Change Summary:
+// - Date / DateTime columns:
+//   - remove `is like` and `is not like`
+//   - migrate `null` or `empty` filters to `blank`
+//   - add `exact date` in comparison_sub_op for existing filters `eq` and `neq`
+// - Year / Time columns:
+//   - remove `is like` and `is not like`
+//   - migrate `null` or `empty` filters to `blank`
+
+function removeLikeAndNlikeFilters(filter: Filter, ncMeta: MetaService) {
+  const actions = [];
+  // remove `is like` and `is not like`
+  if (['like', 'nlike'].includes(filter.comparison_op)) {
+    actions.push(Filter.delete(filter.id, ncMeta));
+  }
+  return actions;
+}
+
+function migrateEqAndNeqFilters(filter: Filter, ncMeta: MetaService) {
+  const actions = [];
+  // remove `is like` and `is not like`
+  if (['eq', 'neq'].includes(filter.comparison_op)) {
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_sub_op: 'exactDate',
+        },
+        ncMeta,
+      ),
+    );
+  }
+  return actions;
+}
+
+function migrateEmptyAndNullFilters(filter: Filter, ncMeta: MetaService) {
+  const actions = [];
+  // remove `is like` and `is not like`
+  if (['empty', 'null'].includes(filter.comparison_op)) {
+    // migrate to blank
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'blank',
+        },
+        ncMeta,
+      ),
+    );
+  } else if (['notempty', 'notnull'].includes(filter.comparison_op)) {
+    // migrate to not blank
+    actions.push(
+      Filter.update(
+        filter.id,
+        {
+          comparison_op: 'notblank',
+        },
+        ncMeta,
+      ),
+    );
+  }
+  return actions;
+}
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const filters = await ncMeta.metaList2(null, null, MetaTable.FILTER_EXP);
+  for (const filter of filters) {
+    if (!filter.fk_column_id || filter.is_group) {
+      continue;
+    }
+    const col = await Column.get({ colId: filter.fk_column_id }, ncMeta);
+    if ([UITypes.Date, UITypes.DateTime].includes(col.uidt)) {
+      await Promise.all([
+        ...removeLikeAndNlikeFilters(filter, ncMeta),
+        ...migrateEmptyAndNullFilters(filter, ncMeta),
+        ...migrateEqAndNeqFilters(filter, ncMeta),
+      ]);
+    } else if ([UITypes.Time, UITypes.Year].includes(col.uidt)) {
+      await Promise.all([
+        ...removeLikeAndNlikeFilters(filter, ncMeta),
+        ...migrateEmptyAndNullFilters(filter, ncMeta),
+      ]);
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncHookUpgrader.ts

@@ -0,0 +1,19 @@
+import { MetaTable } from '../utils/globals';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const actions = [];
+  const hooks = await ncMeta.metaList2(null, null, MetaTable.HOOKS);
+  for (const hook of hooks) {
+    actions.push(
+      ncMeta.metaUpdate(
+        null,
+        null,
+        MetaTable.HOOKS,
+        { version: 'v1' },
+        hook.id,
+      ),
+    );
+  }
+  await Promise.all(actions);
+}

packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader.ts

@@ -0,0 +1,18 @@
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+export default async function (ctx: NcUpgraderCtx) {
+  const projects = await ctx.ncMeta.projectList();
+
+  for (const project of projects) {
+    const projectConfig = JSON.parse(project.config);
+
+    const envVal = projectConfig.envs?.dev;
+    projectConfig.workingEnv = '_noco';
+
+    if (envVal) {
+      projectConfig.envs._noco = envVal;
+      delete projectConfig.envs.dev;
+    }
+    await ctx.ncMeta.projectUpdate(project?.id, projectConfig);
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncProjectEnvUpgrader0011045.ts

@@ -0,0 +1,11 @@
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+export default async function (ctx: NcUpgraderCtx) {
+  const projects = await ctx.ncMeta.projectList();
+
+  for (const project of projects) {
+    const projectConfig = JSON.parse(project.config);
+    projectConfig.env = '_noco';
+    await ctx.ncMeta.projectUpdate(project?.id, projectConfig);
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncProjectRolesUpgrader.ts

@@ -0,0 +1,43 @@
+import { OrgUserRoles } from 'nocodb-sdk';
+import { NC_APP_SETTINGS } from '../constants';
+import Store from '../models/Store';
+import { MetaTable } from '../utils/globals';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+/** Upgrader for upgrading roles */
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const users = await ncMeta.metaList2(null, null, MetaTable.USERS);
+
+  for (const user of users) {
+    user.roles = user.roles
+      .split(',')
+      .map((r) => {
+        // update old role names with new roles
+        if (r === 'user') {
+          return OrgUserRoles.CREATOR;
+        } else if (r === 'user-new') {
+          return OrgUserRoles.VIEWER;
+        }
+        return r;
+      })
+      .join(',');
+    await ncMeta.metaUpdate(
+      null,
+      null,
+      MetaTable.USERS,
+      { roles: user.roles },
+      user.id,
+    );
+  }
+
+  // set invite only signup if user have environment variable set
+  if (process.env.NC_INVITE_ONLY_SIGNUP) {
+    await Store.saveOrUpdate(
+      {
+        value: '{ "invite_only_signup": true }',
+        key: NC_APP_SETTINGS,
+      },
+      ncMeta,
+    );
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncStickyColumnUpgrader.ts

@@ -0,0 +1,134 @@
+import { MetaTable } from '../utils/globals';
+import type { NcUpgraderCtx } from './NcUpgrader';
+
+// before 0.104.3, display value column can be in any position in table
+// with this upgrade we introduced sticky primary column feature
+// this upgrader will make display value column first column in grid views
+
+export default async function ({ ncMeta }: NcUpgraderCtx) {
+  const grid_columns = await ncMeta.metaList2(
+    null,
+    null,
+    MetaTable.GRID_VIEW_COLUMNS,
+  );
+  const grid_views = [...new Set(grid_columns.map((col) => col.fk_view_id))];
+
+  for (const view_id of grid_views) {
+    // get a list of view columns sorted by order
+    const view_columns = await ncMeta.metaList2(
+      null,
+      null,
+      MetaTable.GRID_VIEW_COLUMNS,
+      {
+        condition: {
+          fk_view_id: view_id,
+        },
+        orderBy: {
+          order: 'asc',
+        },
+      },
+    );
+    const view_columns_meta = [];
+
+    // get column meta for each view column
+    for (const col of view_columns) {
+      const col_meta = await ncMeta.metaGet(null, null, MetaTable.COLUMNS, {
+        id: col.fk_column_id,
+      });
+      view_columns_meta.push(col_meta);
+    }
+
+    // if no display value column is set
+    if (!view_columns_meta.some((column) => column.pv)) {
+      const pkIndex = view_columns_meta.findIndex((column) => column.pk);
+
+      // if PK is at the end of table
+      if (pkIndex === view_columns_meta.length - 1) {
+        if (pkIndex > 0) {
+          await ncMeta.metaUpdate(
+            null,
+            null,
+            MetaTable.COLUMNS,
+            { pv: true },
+            view_columns_meta[pkIndex - 1].id,
+          );
+        } else if (view_columns_meta.length > 0) {
+          await ncMeta.metaUpdate(
+            null,
+            null,
+            MetaTable.COLUMNS,
+            { pv: true },
+            view_columns_meta[0].id,
+          );
+        }
+        // pk is not at the end of table
+      } else if (pkIndex > -1) {
+        await ncMeta.metaUpdate(
+          null,
+          null,
+          MetaTable.COLUMNS,
+          { pv: true },
+          view_columns_meta[pkIndex + 1].id,
+        );
+        //  no pk at all
+      } else if (view_columns_meta.length > 0) {
+        await ncMeta.metaUpdate(
+          null,
+          null,
+          MetaTable.COLUMNS,
+          { pv: true },
+          view_columns_meta[0].id,
+        );
+      }
+    }
+
+    const primary_value_column_meta = view_columns_meta.find((col) => col.pv);
+
+    if (primary_value_column_meta) {
+      const primary_value_column = view_columns.find(
+        (col) => col.fk_column_id === primary_value_column_meta.id,
+      );
+      const primary_value_column_index = view_columns.findIndex(
+        (col) => col.fk_column_id === primary_value_column_meta.id,
+      );
+      const view_orders = view_columns.map((col) => col.order);
+      const view_min_order = Math.min(...view_orders);
+
+      // if primary_value_column is not visible, make it visible
+      if (!primary_value_column.show) {
+        await ncMeta.metaUpdate(
+          null,
+          null,
+          MetaTable.GRID_VIEW_COLUMNS,
+          { show: true },
+          primary_value_column.id,
+        );
+      }
+
+      if (
+        primary_value_column.order === view_min_order &&
+        view_orders.filter((o) => o === view_min_order).length === 1
+      ) {
+        // if primary_value_column is in first order do nothing
+        continue;
+      } else {
+        // if primary_value_column not in first order, move it to the start of array
+        if (primary_value_column_index !== 0) {
+          const temp_pv = view_columns.splice(primary_value_column_index, 1);
+          view_columns.unshift(...temp_pv);
+        }
+
+        // update order of all columns in view to match the order in array
+        for (let i = 0; i < view_columns.length; i++) {
+          await ncMeta.metaUpdate(
+            null,
+            null,
+            MetaTable.GRID_VIEW_COLUMNS,
+            { order: i + 1 },
+            view_columns[i].id,
+          );
+        }
+      }
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/ncUpgradeErrors.ts

@@ -0,0 +1,27 @@
+export function throwTimeoutError(e, timeoutErrorInfo) {
+  if (
+    [
+      'EHOSTDOWN',
+      'ETIMEDOUT',
+      'EHOSTUNREACH',
+      'ENOTFOUND',
+      'ECONNREFUSED',
+    ].includes(e.code)
+  ) {
+    let db = '';
+    if (timeoutErrorInfo.connection.filename) {
+      // for sqlite
+      db = timeoutErrorInfo.connection.filename;
+    } else if (
+      timeoutErrorInfo.connection.database &&
+      timeoutErrorInfo.connection.host &&
+      timeoutErrorInfo.connection.port
+    ) {
+      db = `${timeoutErrorInfo.connection.database} (${timeoutErrorInfo.connection.host}:${timeoutErrorInfo.connection.port})`;
+    }
+    throw new Error(
+      `Failed to connect the database ${db} for Project ${timeoutErrorInfo.projectTitle}.
+       Please fix the connection issue or remove the project before trying to upgrade.`
+    );
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/BaseApiBuilder.ts

@@ -0,0 +1,611 @@
+import debug from 'debug';
+import { Router } from 'express';
+import { BaseModelSql } from '../../db/BaseModelSql';
+import { XKnex } from '../../db/CustomKnex';
+import ModelXcMetaFactory from '../../db/sql-mgr/code/models/xc/ModelXcMetaFactory'
+import { DbConfig, NcConfig } from '../../interface/config';
+import inflection from 'inflection';
+import { MetaService } from '../../meta/meta.service';
+import NcConnectionMgr from '../../utils/common/NcConnectionMgr';
+import ncModelsOrderUpgrader from './jobs/ncModelsOrderUpgrader';
+import ncParentModelTitleUpgrader from './jobs/ncParentModelTitleUpgrader';
+import ncRemoveDuplicatedRelationRows from './jobs/ncRemoveDuplicatedRelationRows';
+import type Noco from '../../Noco';
+import type NcProjectBuilder from './NcProjectBuilder';
+import type { MysqlClient, PgClient, SqlClient } from 'nc-help';
+
+const log = debug('nc:api:base');
+
+export default abstract class BaseApiBuilder<T extends Noco> {
+  public abstract readonly type: string;
+
+  public get knex(): XKnex {
+    return this.sqlClient?.knex || this.dbDriver;
+  }
+
+  public get prefix() {
+    return this.projectBuilder?.prefix;
+  }
+
+  public get apiType(): string {
+    return this.connectionConfig?.meta?.api?.type;
+  }
+
+  public get apiPrefix(): string {
+    return this.connectionConfig?.meta?.api?.prefix;
+  }
+
+  public get dbAlias(): any {
+    return this.connectionConfig?.meta?.dbAlias;
+  }
+
+  public get router(): Router {
+    if (!this.apiRouter) {
+      this.baseLog(`router : Initializing builder router`);
+      this.apiRouter = Router();
+      // (this.app as any).router.use('/', this.apiRouter)
+      (this.projectBuilder as any).router.use('/', this.apiRouter);
+    }
+    return this.apiRouter;
+  }
+
+  public get routeVersionLetter(): string {
+    return this.connectionConfig?.meta?.api?.prefix || 'v1';
+  }
+
+  protected get projectId(): string {
+    return this.projectBuilder?.id;
+  }
+
+  public get xcModels() {
+    return this.models;
+  }
+
+  public get client() {
+    return this.connectionConfig?.client;
+  }
+
+  public readonly app: T;
+
+  public hooks: {
+    [tableName: string]: {
+      [key: string]: Array<{
+        event: string;
+        url: string;
+        [key: string]: any;
+      }>;
+    };
+  };
+
+  public formViews: {
+    [tableName: string]: any;
+  };
+
+  protected tablesCount = 0;
+  protected relationsCount = 0;
+  protected viewsCount = 0;
+  protected functionsCount = 0;
+  protected proceduresCount = 0;
+
+  protected projectBuilder: NcProjectBuilder;
+
+  protected models: { [key: string]: BaseModelSql };
+
+  protected metas: { [key: string]: NcMetaData };
+
+  protected sqlClient: MysqlClient | PgClient | SqlClient | any;
+
+  protected dbDriver: XKnex;
+  protected config: NcConfig;
+  protected connectionConfig: DbConfig;
+
+  protected procedureOrFunctionAcls: {
+    [name: string]: { [role: string]: boolean };
+  };
+  protected xcMeta: MetaService;
+
+  private apiRouter: Router;
+
+  constructor(
+    app: T,
+    projectBuilder: NcProjectBuilder,
+    config: NcConfig,
+    connectionConfig: DbConfig,
+  ) {
+    this.models = {};
+    this.app = app;
+    this.config = config;
+    this.connectionConfig = connectionConfig;
+    this.metas = {};
+    this.procedureOrFunctionAcls = {};
+    this.hooks = {};
+    this.formViews = {};
+    this.projectBuilder = projectBuilder;
+  }
+
+  public getDbType(): any {
+    return this.connectionConfig.client;
+  }
+
+  public getDbName(): any {
+    return (this.connectionConfig.connection as any)?.database;
+  }
+
+  public getDbAlias(): any {
+    return this.connectionConfig?.meta?.dbAlias;
+  }
+
+  public async getSqlClient() {
+    return NcConnectionMgr.getSqlClient({
+      dbAlias: this.dbAlias,
+      env: this.config.env,
+      config: this.config,
+      projectId: this.projectId,
+    });
+  }
+
+  public async xcUpgrade(): Promise<any> {
+    const NC_VERSIONS = [
+      { name: '0009000', handler: null },
+      { name: '0009044', handler: this.ncUpManyToMany.bind(this) },
+      { name: '0083006', handler: ncModelsOrderUpgrader },
+      { name: '0083007', handler: ncParentModelTitleUpgrader },
+      { name: '0083008', handler: ncRemoveDuplicatedRelationRows },
+      { name: '0084002', handler: this.ncUpAddNestedResolverArgs.bind(this) },
+    ];
+    if (!(await this.xcMeta?.knex?.schema?.hasTable?.('nc_store'))) {
+      return;
+    }
+    this.baseLog(`xcUpgrade : Getting configuration from meta database`);
+
+    const config = await this.xcMeta.metaGet(
+      this.projectId,
+      this.dbAlias,
+      'nc_store',
+      { key: 'NC_CONFIG' },
+    );
+
+    if (config) {
+      const configObj: NcConfig = JSON.parse(config.value);
+      if (configObj.version !== process.env.NC_VERSION) {
+        for (const version of NC_VERSIONS) {
+          // compare current version and old version
+          if (version.name > configObj.version) {
+            this.baseLog(
+              `xcUpgrade : Upgrading '%s' => '%s'`,
+              configObj.version,
+              version.name,
+            );
+            await version?.handler?.(<NcBuilderUpgraderCtx>{
+              xcMeta: this.xcMeta,
+              builder: this,
+              dbAlias: this.dbAlias,
+              projectId: this.projectId,
+            });
+
+            // update version in meta after each upgrade
+            configObj.version = version.name;
+            await this.xcMeta.metaUpdate(
+              this.projectId,
+              this.dbAlias,
+              'nc_store',
+              {
+                value: JSON.stringify(configObj),
+              },
+              {
+                key: 'NC_CONFIG',
+              },
+            );
+
+            // todo: backup data
+          }
+          if (version.name === process.env.NC_VERSION) {
+            break;
+          }
+        }
+        configObj.version = process.env.NC_VERSION;
+        await this.xcMeta.metaUpdate(
+          this.projectId,
+          this.dbAlias,
+          'nc_store',
+          {
+            value: JSON.stringify(configObj),
+          },
+          {
+            key: 'NC_CONFIG',
+          },
+        );
+      }
+    } else {
+      this.baseLog(`xcUpgrade : Inserting config to meta database`);
+      const configObj: NcConfig = JSON.parse(JSON.stringify(this.config));
+      delete configObj.envs;
+      const isOld = (
+        await this.xcMeta.metaList(this.projectId, this.dbAlias, 'nc_models')
+      )?.length;
+      configObj.version = isOld ? '0009000' : process.env.NC_VERSION;
+      await this.xcMeta.metaInsert(this.projectId, this.dbAlias, 'nc_store', {
+        key: 'NC_CONFIG',
+        value: JSON.stringify(configObj),
+      });
+      if (isOld) {
+        await this.xcUpgrade();
+      }
+    }
+  }
+
+  public getProjectId(): string {
+    return this.projectId;
+  }
+
+  public async init(): Promise<void> {
+    await this.xcUpgrade();
+  }
+
+  protected async initDbDriver(): Promise<void> {
+    this.dbDriver = await NcConnectionMgr.get({
+      dbAlias: this.dbAlias,
+      env: this.config.env,
+      config: this.config,
+      projectId: this.projectId,
+    });
+    this.sqlClient = await NcConnectionMgr.getSqlClient({
+      dbAlias: this.dbAlias,
+      env: this.config.env,
+      config: this.config,
+      projectId: this.projectId,
+    });
+  }
+
+  private baseLog(str, ...args): void {
+    log(`${this.dbAlias} : ${str}`, ...args);
+  }
+
+  protected generateContextForTable(
+    tn: string,
+    columns: any[],
+    relations,
+    hasMany: any[],
+    belongsTo: any[],
+    type = 'table',
+    tableNameAlias?: string,
+  ): any {
+    this.baseLog(`generateContextForTable : '%s' %s`, tn, type);
+
+    for (const col of columns) {
+      col._cn = col._cn || this.getColumnNameAlias(col);
+    }
+
+    // tslint:disable-next-line:variable-name
+    const _tn = tableNameAlias || this.getTableNameAlias(tn);
+
+    const ctx = {
+      dbType: this.connectionConfig.client,
+      tn,
+      _tn,
+      tn_camelize: inflection.camelize(_tn),
+      tn_camelize_low: inflection.camelize(_tn, true),
+      columns,
+      relations,
+      hasMany,
+      belongsTo,
+      dbAlias: '',
+      routeVersionLetter: this.routeVersionLetter,
+      type,
+      project_id: this.projectId,
+    };
+    return ctx;
+  }
+
+  private getColumnNameAlias(col, tableName?: string) {
+    return (
+      this.metas?.[tableName]?.columns?.find((c) => c.cn === col.cn)?._cn ||
+      col._cn ||
+      this.getInflectedName(col.cn, this.connectionConfig?.meta?.inflection?.cn)
+    );
+  }
+
+  // table alias functions
+  protected getInflectedName(_name: string, inflectionFns: string): string {
+    let name = _name;
+    if (process.env.NC_INFLECTION) {
+      inflectionFns = 'camelize';
+    }
+
+    if (inflectionFns && inflectionFns !== 'none') {
+      name = inflectionFns
+        .split(',')
+        .reduce((out, fn) => inflection?.[fn]?.(out) || out, name);
+    }
+    return this.apiType === 'graphql' ? name.replace(/[^_\da-z]/gi, '_') : name;
+  }
+
+  protected async ncUpAddNestedResolverArgs(_ctx: any): Promise<any> {}
+
+  protected getTableNameAlias(tableName: string) {
+    let tn = tableName;
+    if (this.metas?.[tn]?._tn) {
+      return this.metas?.[tn]?._tn;
+    }
+
+    if (this.projectBuilder?.prefix) {
+      tn = tn.replace(this.projectBuilder?.prefix, '');
+    }
+
+    const modifiedTableName = tn?.replace(/^(?=\d+)/, 'ISN___');
+    return this.getInflectedName(
+      modifiedTableName,
+      this.connectionConfig?.meta?.inflection?.tn,
+    );
+  }
+
+  protected async ncUpManyToMany(_ctx: any): Promise<any> {
+    const models = await this.xcMeta.metaList(
+      this.projectId,
+      this.dbAlias,
+      'nc_models',
+      {
+        fields: ['meta'],
+        condition: {
+          type: 'table',
+        },
+      },
+    );
+    if (!models.length) {
+      return;
+    }
+    const metas = [];
+    // add virtual columns for relations
+    for (const metaObj of models) {
+      const meta = JSON.parse(metaObj.meta);
+      metas.push(meta);
+      const ctx = this.generateContextForTable(
+        meta.tn,
+        meta.columns,
+        [],
+        meta.hasMany,
+        meta.belongsTo,
+        meta.type,
+        meta._tn,
+      );
+      // generate virtual columns
+      meta.v = ModelXcMetaFactory.create(this.connectionConfig, {
+        dir: '',
+        ctx,
+        filename: '',
+      }).getVitualColumns();
+      // set default display values
+      ModelXcMetaFactory.create(
+        this.connectionConfig,
+        {},
+      ).mapDefaultDisplayValue(meta.columns);
+      // update meta
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          meta: JSON.stringify(meta),
+        },
+        { title: meta.tn },
+      );
+    }
+
+    // generate many to many relations an columns
+    await this.getManyToManyRelations({ localMetas: metas });
+    return metas;
+  }
+
+  protected async getManyToManyRelations({
+    parent = null,
+    child = null,
+    localMetas = null,
+  } = {}): Promise<Set<any>> {
+    const metas = new Set<any>();
+    const assocMetas = new Set<any>();
+
+    if (localMetas) {
+      for (const meta of localMetas) {
+        this.metas[meta.tn] = meta;
+      }
+    }
+
+    for (const meta of Object.values(this.metas)) {
+      // check if table is a Bridge table(or Associative Table) by checking
+      // number of foreign keys and columns
+      if (meta.belongsTo?.length === 2 && meta.columns.length < 5) {
+        if (
+          parent &&
+          child &&
+          !(
+            [parent, child].includes(meta.belongsTo[0].rtn) &&
+            [parent, child].includes(meta.belongsTo[1].rtn)
+          )
+        ) {
+          continue;
+        }
+
+        const tableMetaA = this.metas[meta.belongsTo[0].rtn];
+        const tableMetaB = this.metas[meta.belongsTo[1].rtn];
+
+        /*        // remove hasmany relation with associate table from tables
+                tableMetaA.hasMany.splice(tableMetaA.hasMany.findIndex(hm => hm.tn === meta.tn), 1)
+                tableMetaB.hasMany.splice(tableMetaB.hasMany.findIndex(hm => hm.tn === meta.tn), 1)*/
+
+        // add manytomany data under metadata of both linked tables
+        tableMetaA.manyToMany = tableMetaA.manyToMany || [];
+        if (tableMetaA.manyToMany.every((mm) => mm.vtn !== meta.tn)) {
+          tableMetaA.manyToMany.push({
+            tn: tableMetaA.tn,
+            cn: meta.belongsTo[0].rcn,
+            vtn: meta.tn,
+            vcn: meta.belongsTo[0].cn,
+            vrcn: meta.belongsTo[1].cn,
+            rtn: meta.belongsTo[1].rtn,
+            rcn: meta.belongsTo[1].rcn,
+            _tn: tableMetaA._tn,
+            _cn: meta.belongsTo[0]._rcn,
+            _rtn: meta.belongsTo[1]._rtn,
+            _rcn: meta.belongsTo[1]._rcn,
+          });
+          metas.add(tableMetaA);
+        }
+        // ignore if A & B are same table
+        if (tableMetaB !== tableMetaA) {
+          tableMetaB.manyToMany = tableMetaB.manyToMany || [];
+          if (tableMetaB.manyToMany.every((mm) => mm.vtn !== meta.tn)) {
+            tableMetaB.manyToMany.push({
+              tn: tableMetaB.tn,
+              cn: meta.belongsTo[1].rcn,
+              vtn: meta.tn,
+              vcn: meta.belongsTo[1].cn,
+              vrcn: meta.belongsTo[0].cn,
+              rtn: meta.belongsTo[0].rtn,
+              rcn: meta.belongsTo[0].rcn,
+              _tn: tableMetaB._tn,
+              _cn: meta.belongsTo[1]._rcn,
+              _rtn: meta.belongsTo[0]._rtn,
+              _rcn: meta.belongsTo[0]._rcn,
+            });
+            metas.add(tableMetaB);
+          }
+        }
+        assocMetas.add(meta);
+      }
+    }
+
+    // Update metadata of tables which have manytomany relation
+    // and recreate basemodel with new meta information
+    for (const meta of metas) {
+      let queryParams;
+
+      // update showfields on new many to many relation create
+      if (parent && child) {
+        try {
+          queryParams = JSON.parse(
+            (
+              await this.xcMeta.metaGet(
+                this.projectId,
+                this.dbAlias,
+                'nc_models',
+                { title: meta.tn },
+              )
+            ).query_params,
+          );
+        } catch (e) {
+          //  ignore
+        }
+      }
+
+      meta.v = [
+        ...meta.v.filter(
+          (vc) => !(vc.hm && meta.manyToMany.some((mm) => vc.hm.tn === mm.vtn)),
+        ),
+        // todo: ignore duplicate m2m relations
+        // todo: optimize, just compare associative table(Vtn)
+        ...meta.manyToMany
+          .filter(
+            (v, i) =>
+              !meta.v.some(
+                (v1) =>
+                  v1.mm &&
+                  ((v1.mm.tn === v.tn && v.rtn === v1.mm.rtn) ||
+                    (v1.mm.rtn === v.tn && v.tn === v1.mm.rtn)) &&
+                  v.vtn === v1.mm.vtn,
+              ) &&
+              // ignore duplicate
+              !meta.manyToMany.some(
+                (v1, i1) =>
+                  i1 !== i &&
+                  v1.tn === v.tn &&
+                  v.rtn === v1.rtn &&
+                  v.vtn === v1.vtn,
+              ),
+          )
+          .map((mm) => {
+            if (
+              queryParams?.showFields &&
+              !(`${mm._tn} <=> ${mm._rtn}` in queryParams.showFields)
+            ) {
+              queryParams.showFields[`${mm._tn} <=> ${mm._rtn}`] = true;
+            }
+
+            return {
+              mm,
+              _cn: `${mm._tn} <=> ${mm._rtn}`,
+            };
+          }),
+      ];
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          meta: JSON.stringify(meta),
+          ...(queryParams ? { query_params: JSON.stringify(queryParams) } : {}),
+        },
+        { title: meta.tn },
+      );
+      // XcCache.del([this.projectId, this.dbAlias, 'table', meta.tn].join('::'));
+      // if (!localMetas) {
+      //   this.models[meta.tn] = this.getBaseModel(meta);
+      // }
+    }
+
+    // Update metadata of associative table
+    for (const meta of assocMetas) {
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          mm: 1,
+        },
+        { title: meta.tn },
+      );
+    }
+
+    return metas;
+  }
+
+
+}
+
+interface NcBuilderUpgraderCtx {
+  xcMeta: MetaService;
+  builder: BaseApiBuilder<any>;
+  projectId: string;
+  dbAlias: string;
+}
+
+interface NcMetaData {
+  tn: string;
+  _tn?: string;
+  v: Array<{
+    _cn?: string;
+    [key: string]: any;
+  }>;
+  columns: Array<{
+    _cn?: string;
+    cn?: string;
+    uidt?: string;
+    [key: string]: any;
+  }>;
+
+  [key: string]: any;
+}
+
+type XcTablesPopulateParams = {
+  tableNames?: Array<{
+    tn: string;
+    _tn?: string;
+  }>;
+  type?: 'table' | 'view' | 'function' | 'procedure';
+  columns?: {
+    [tn: string]: any;
+  };
+  oldMetas?: {
+    [tn: string]: NcMetaData;
+  };
+};
+export { NcBuilderUpgraderCtx, NcMetaData, XcTablesPopulateParams };

packages/nocodb-nest/src/version-upgrader/v1-legacy/NcProjectBuilder.ts

@@ -0,0 +1,198 @@
+import { Router } from 'express';
+import { SqlClientFactory } from '../../db/sql-client/lib/SqlClientFactory';
+import Noco from '../../Noco';
+import { GqlApiBuilder } from './gql/GqlApiBuilder';
+import { RestApiBuilder } from './rest/RestApiBuilder';
+import type { NcConfig } from '../../interface/config';
+
+export default class NcProjectBuilder {
+  public readonly id: string;
+  public readonly title: string;
+  public readonly description: string;
+  public readonly router: Router;
+  public readonly apiBuilders: Array<RestApiBuilder | GqlApiBuilder> = [];
+  private _config: any;
+
+  protected startTime;
+  protected app: Noco;
+  protected appConfig: NcConfig;
+  protected apiInfInfoList: any[] = [];
+  protected aggregatedApiInfo: any;
+  protected authHook: any;
+
+  constructor(app: Noco, appConfig: NcConfig, project: any) {
+    this.app = app;
+    this.appConfig = appConfig;
+
+    if (project) {
+      this.id = project.id;
+      this.title = project.title;
+      this.description = project.description;
+      this._config = { ...this.appConfig, ...JSON.parse(project.config) };
+      this.router = Router();
+    }
+  }
+
+  public async init(_isFirstTime?: boolean) {
+    try {
+      // await this.addAuthHookToMiddleware();
+
+      this.startTime = Date.now();
+      const allRoutesInfo: any[] = [];
+      // await this.app.ncMeta.projectStatusUpdate(this.id, 'starting');
+      // await this.syncMigration();
+      await this._createApiBuilder();
+      // this.initApiInfoRoute();
+
+      /* Create REST APIs / GraphQL Resolvers */
+      for (const meta of this.apiBuilders) {
+        let routeInfo;
+        if (meta instanceof RestApiBuilder) {
+          console.log(
+            `Creating REST APIs ${meta.getDbType()} - > ${meta.getDbName()}`,
+          );
+          routeInfo = await (meta as RestApiBuilder).init();
+        } else if (meta instanceof GqlApiBuilder) {
+          console.log(
+            `Creating GraphQL APIs ${meta.getDbType()} - > ${meta.getDbName()}`,
+          );
+          routeInfo = await (meta as GqlApiBuilder).init();
+        }
+        allRoutesInfo.push(routeInfo);
+        // this.progress(routeInfo, allRoutesInfo, isFirstTime);
+      }
+
+      // this.app.projectRouter.use(`/nc/${this.id}`, this.router);
+      // await this.app.ncMeta.projectStatusUpdate(this.id, 'started');
+    } catch (e) {
+      console.log(e);
+      throw e;
+      // await this.app.ncMeta.projectStatusUpdate(this.id, 'stopped');
+    }
+  }
+
+  protected async _createApiBuilder() {
+    this.apiBuilders.splice(0, this.apiBuilders.length);
+    let i = 0;
+
+    const connectionConfigs = [];
+
+    /* for each db create an api builder */
+    for (const db of this.config?.envs?.[this.appConfig?.workingEnv]?.db ||
+      []) {
+      let Builder;
+      switch (db.meta.api.type) {
+        case 'graphql':
+          Builder = GqlApiBuilder;
+          break;
+
+        case 'rest':
+          Builder = RestApiBuilder;
+          break;
+      }
+
+      if ((db?.connection as any)?.database) {
+        const connectionConfig = {
+          ...db,
+          meta: {
+            ...db.meta,
+            api: {
+              ...db.meta.api,
+              prefix: db.meta.api.prefix || this.genVer(i),
+            },
+          },
+        };
+
+        this.apiBuilders.push(
+          new Builder(
+            this.app,
+            this,
+            this.config,
+            connectionConfig,
+            this.app.ncMeta,
+          ),
+        );
+        connectionConfigs.push(connectionConfig);
+        i++;
+      } else if (db.meta?.allSchemas) {
+        /* get all schemas and create APIs for all of them */
+        const sqlClient = await SqlClientFactory.create({
+          ...db,
+          connection: { ...db.connection, database: undefined },
+        });
+
+        const schemaList = (await sqlClient.schemaList({}))?.data?.list;
+        for (const schema of schemaList) {
+          const connectionConfig = {
+            ...db,
+            connection: { ...db.connection, database: schema.schema_name },
+            meta: {
+              ...db.meta,
+              dbAlias: i ? db.meta.dbAlias + i : db.meta.dbAlias,
+              api: {
+                ...db.meta.api,
+                prefix: db.meta.api.prefix || this.genVer(i),
+              },
+            },
+          };
+
+          this.apiBuilders.push(
+            new Builder(
+              this.app,
+              this,
+              this.config,
+              connectionConfig,
+              this.app.ncMeta,
+            ),
+          );
+          connectionConfigs.push(connectionConfig);
+
+          i++;
+        }
+
+        sqlClient.knex.destroy();
+      }
+    }
+    if (this.config?.envs?.[this.appConfig.workingEnv]?.db) {
+      this.config.envs[this.appConfig.workingEnv].db.splice(
+        0,
+        this.config.envs[this.appConfig.workingEnv].db.length,
+        ...connectionConfigs,
+      );
+    }
+  }
+
+  protected genVer(i): string {
+    const l = 'vwxyzabcdefghijklmnopqrstu';
+    return (
+      i
+        .toString(26)
+        .split('')
+        .map((v) => l[parseInt(v, 26)])
+        .join('') + '1'
+    );
+  }
+
+  protected static triggerGarbageCollect() {
+    try {
+      if (global.gc) {
+        global.gc();
+      }
+    } catch (e) {
+      console.log('`node --expose-gc index.js`');
+      process.exit();
+    }
+  }
+
+  public get prefix(): string {
+    return this.config?.prefix;
+  }
+
+  public get config(): any {
+    return this._config;
+  }
+
+  public updateConfig(config: string) {
+    this._config = { ...this.appConfig, ...JSON.parse(config) };
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/gql/GqlApiBuilder.ts

@@ -0,0 +1,149 @@
+import debug from 'debug';
+import { Router } from 'express';
+import GqlXcSchemaFactory from '../../../../../nocodb/src/lib/db/sql-mgr/code/gql-schema/xc-ts/GqlXcSchemaFactory';
+import { MetaService } from '../../../meta/meta.service';
+import Noco from '../../../Noco';
+import type NcProjectBuilder from '../NcProjectBuilder';
+import type { DbConfig, NcConfig } from '../../../interface/config';
+import type XcMetaMgr from '../../../interface/XcMetaMgr';
+import BaseApiBuilder from '../BaseApiBuilder';
+
+export class GqlApiBuilder extends BaseApiBuilder<Noco> implements XcMetaMgr {
+  public readonly type = 'gql';
+
+  private readonly gqlRouter: Router;
+
+  constructor(
+    app: Noco,
+    projectBuilder: NcProjectBuilder,
+    config: NcConfig,
+    connectionConfig: DbConfig,
+    xcMeta?: MetaService,
+  ) {
+    super(app, projectBuilder, config, connectionConfig);
+    this.config = config;
+    this.connectionConfig = connectionConfig;
+    this.gqlRouter = Router();
+    this.xcMeta = xcMeta;
+  }
+
+  public async init(): Promise<void> {
+    await super.init();
+  }
+
+  protected async ncUpAddNestedResolverArgs(_ctx: any): Promise<any> {
+    const models = await this.xcMeta.metaList(
+      this.projectId,
+      this.dbAlias,
+      'nc_models',
+      {
+        fields: ['meta'],
+        condition: {
+          type: 'table',
+        },
+      },
+    );
+    if (!models.length) {
+      return;
+    }
+    // add virtual columns for relations
+    for (const metaObj of models) {
+      const meta = JSON.parse(metaObj.meta);
+      const ctx = this.generateContextForTable(
+        meta.tn,
+        meta.columns,
+        [],
+        meta.hasMany,
+        meta.belongsTo,
+        meta.type,
+        meta._tn,
+      );
+
+      /* generate gql schema of the table */
+      const schema = GqlXcSchemaFactory.create(this.connectionConfig, {
+        dir: '',
+        ctx: {
+          ...ctx,
+          manyToMany: meta.manyToMany,
+        },
+        filename: '',
+      }).getString();
+
+      /* update schema in metadb */
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          schema,
+        },
+        {
+          title: meta.tn,
+          type: 'table',
+        },
+      );
+    }
+  }
+
+  protected async ncUpManyToMany(ctx: any): Promise<any> {
+    const metas = await super.ncUpManyToMany(ctx);
+
+    if (!metas) {
+      return;
+    }
+    for (const meta of metas) {
+      const ctx = this.generateContextForTable(
+        meta.tn,
+        meta.columns,
+        [],
+        meta.hasMany,
+        meta.belongsTo,
+        meta.type,
+        meta._tn,
+      );
+
+      /* generate gql schema of the table */
+      const schema = GqlXcSchemaFactory.create(this.connectionConfig, {
+        dir: '',
+        ctx: {
+          ...ctx,
+          manyToMany: meta.manyToMany,
+        },
+        filename: '',
+      }).getString();
+
+      /* update schema in metadb */
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          schema,
+        },
+        {
+          title: meta.tn,
+          type: 'table',
+        },
+      );
+
+      // todo : add loaders
+
+      if (meta.manyToMany) {
+        for (const mm of meta.manyToMany) {
+          await this.xcMeta.metaInsert(
+            this.projectId,
+            this.dbAlias,
+            'nc_loaders',
+            {
+              title: `${mm.tn}Mm${mm.rtn}List`,
+              parent: mm.tn,
+              child: mm.rtn,
+              relation: 'mm',
+              resolver: 'mmlist',
+            },
+          );
+        }
+      }
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncModelsOrderUpgrader.ts

@@ -0,0 +1,48 @@
+import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder';
+
+export default async function (ctx: NcBuilderUpgraderCtx) {
+  const models = await ctx.xcMeta.metaList(
+    ctx.projectId,
+    ctx.dbAlias,
+    'nc_models',
+    {
+      xcCondition: {
+        _or: [{ type: { eq: 'table' } }, { type: { eq: 'view' } }],
+      },
+    },
+  );
+  let order = 0;
+  for (const model of models) {
+    await ctx.xcMeta.metaUpdate(
+      ctx.projectId,
+      ctx.dbAlias,
+      'nc_models',
+      {
+        order: ++order,
+        view_order: 1,
+      },
+      model.id,
+    );
+
+    const views = await ctx.xcMeta.metaList(
+      ctx.projectId,
+      ctx.dbAlias,
+      'nc_models',
+      {
+        condition: { parent_model_title: model.title },
+      },
+    );
+    let view_order = 1;
+    for (const view of views) {
+      await ctx.xcMeta.metaUpdate(
+        ctx.projectId,
+        ctx.dbAlias,
+        'nc_models',
+        {
+          view_order: ++view_order,
+        },
+        view.id,
+      );
+    }
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncParentModelTitleUpgrader.ts

@@ -0,0 +1,29 @@
+import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder';
+
+export default async function (ctx: NcBuilderUpgraderCtx) {
+  const views = await ctx.xcMeta.metaList(
+    ctx.projectId,
+    ctx.dbAlias,
+    'nc_models',
+    {
+      condition: {
+        type: 'vtable',
+      },
+    },
+  );
+
+  for (const view of views) {
+    await ctx.xcMeta.metaUpdate(
+      ctx.projectId,
+      ctx.dbAlias,
+      'nc_disabled_models_for_role',
+      {
+        parent_model_title: view.parent_model_title,
+      },
+      {
+        type: 'vtable',
+        title: view.title,
+      },
+    );
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/jobs/ncRemoveDuplicatedRelationRows.ts

@@ -0,0 +1,113 @@
+import type { NcBuilderUpgraderCtx } from '../BaseApiBuilder';
+
+export default async function (ctx: NcBuilderUpgraderCtx) {
+  try {
+    const relations = await ctx.xcMeta.metaList(
+      ctx.projectId,
+      ctx.dbAlias,
+      'nc_relations',
+    );
+
+    const duplicates = [];
+
+    for (const relation of relations) {
+      if (relation.type !== 'real' || duplicates.includes(relation)) continue;
+      const duplicateRelIndex = relations.findIndex(
+        (rel) =>
+          rel !== relation &&
+          rel.tn === relation.tn &&
+          rel.rtn === relation.rtn &&
+          rel.cn === relation.cn &&
+          rel.rcn === relation.rcn &&
+          rel.type === 'real',
+      );
+
+      if (duplicateRelIndex > -1) duplicates.push(relations[duplicateRelIndex]);
+    }
+
+    // delete relation
+    for (const dupRelation of duplicates) {
+      await ctx.xcMeta.metaDelete(
+        ctx.projectId,
+        ctx.dbAlias,
+        'nc_relations',
+        dupRelation.id,
+      );
+      {
+        const tnModel = await ctx.xcMeta.metaGet(
+          ctx.projectId,
+          ctx.dbAlias,
+          'nc_models',
+          {
+            type: 'table',
+            title: dupRelation.tn,
+          },
+        );
+
+        const meta = JSON.parse(tnModel.meta);
+
+        const duplicateBts = meta.belongsTo.filter(
+          (bt) =>
+            bt.tn === dupRelation.tn &&
+            bt.rtn === dupRelation.rtn &&
+            bt.cn === dupRelation.cn &&
+            bt.rcn === dupRelation.rcn &&
+            bt.type === 'real',
+        );
+
+        if (duplicateBts?.length > 1) {
+          meta.belongsTo.splice(meta.belongsTo.indexOf(duplicateBts[1]), 1);
+        }
+
+        await ctx.xcMeta.metaUpdate(
+          ctx.projectId,
+          ctx.dbAlias,
+          'nc_models',
+          { meta: JSON.stringify(meta) },
+          {
+            type: 'table',
+            title: dupRelation.tn,
+          },
+        );
+      }
+      {
+        const rtnModel = await ctx.xcMeta.metaGet(
+          ctx.projectId,
+          ctx.dbAlias,
+          'nc_models',
+          {
+            type: 'table',
+            title: dupRelation.rtn,
+          },
+        );
+
+        const meta = JSON.parse(rtnModel.meta);
+
+        const duplicateHms = meta.hasMany.filter(
+          (bt) =>
+            bt.tn === dupRelation.tn &&
+            bt.rtn === dupRelation.rtn &&
+            bt.cn === dupRelation.cn &&
+            bt.rcn === dupRelation.rcn &&
+            bt.type === 'real',
+        );
+
+        if (duplicateHms?.length > 1) {
+          meta.hasMany.splice(meta.hasMany.indexOf(duplicateHms[1]), 1);
+        }
+        await ctx.xcMeta.metaUpdate(
+          ctx.projectId,
+          ctx.dbAlias,
+          'nc_models',
+          { meta: JSON.stringify(meta) },
+          {
+            type: 'table',
+            title: dupRelation.rtn,
+          },
+        );
+      }
+    }
+  } catch (e) {
+    console.log(e);
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/rest/RestApiBuilder.ts

@@ -0,0 +1,178 @@
+import { Router } from 'express';
+import debug from 'debug';
+import autoBind from 'auto-bind';
+import SwaggerXc from '../../../db/sql-mgr/code/routers/xc-ts/SwaggerXc'
+import ExpressXcTsRoutes from '../../../db/sql-mgr/code/routes/xc-ts/ExpressXcTsRoutes'
+import { MetaService } from '../../../meta/meta.service';
+import Noco from '../../../Noco';
+import NcHelp from '../../../utils/NcHelp'
+import type { DbConfig, NcConfig } from '../../../interface/config';
+import type NcProjectBuilder from '../NcProjectBuilder';
+import BaseApiBuilder, { XcTablesPopulateParams } from '../BaseApiBuilder';
+
+const log = debug('nc:api:rest');
+
+export class RestApiBuilder extends BaseApiBuilder<Noco> {
+  public readonly type = 'rest';
+
+
+  protected nocoTypes: any;
+  protected nocoRootResolvers: any;
+  private routers: { [key: string]: Router };
+
+  constructor(
+    app: Noco,
+    projectBuilder: NcProjectBuilder,
+    config: NcConfig,
+    connectionConfig: DbConfig,
+    xcMeta?: MetaService,
+  ) {
+    super(app, projectBuilder, config, connectionConfig);
+    autoBind(this);
+    this.routers = {};
+    this.hooks = {};
+    this.xcMeta = xcMeta;
+  }
+
+  public async init(): Promise<void> {
+    await super.init();
+  }
+
+  protected async ncUpManyToMany(ctx: any): Promise<any> {
+    const metas = await super.ncUpManyToMany(ctx);
+    if (!metas) {
+      return;
+    }
+    for (const meta of metas) {
+      const ctx = this.generateContextForTable(
+        meta.tn,
+        meta.columns,
+        [],
+        meta.hasMany,
+        meta.belongsTo,
+        meta.type,
+        meta._tn
+      );
+
+      /* create routes for table */
+      const routes = new ExpressXcTsRoutes({
+        dir: '',
+        ctx,
+        filename: '',
+      }).getObjectWithoutFunctions();
+
+      /* create nc_routes, add new routes or update order */
+      const routesInsertion = routes.map((route, i) => {
+        return async () => {
+          if (
+            !(await this.xcMeta.metaGet(
+              this.projectId,
+              this.dbAlias,
+              'nc_routes',
+              {
+                path: route.path,
+                tn: meta.tn,
+                title: meta.tn,
+                type: route.type,
+              }
+            ))
+          ) {
+            await this.xcMeta.metaInsert(
+              this.projectId,
+              this.dbAlias,
+              'nc_routes',
+              {
+                acl: JSON.stringify(route.acl),
+                handler: JSON.stringify(route.handler),
+                order: i,
+                path: route.path,
+                tn: meta.tn,
+                title: meta.tn,
+                type: route.type,
+              }
+            );
+          } else {
+            await this.xcMeta.metaUpdate(
+              this.projectId,
+              this.dbAlias,
+              'nc_routes',
+              {
+                order: i,
+              },
+              {
+                path: route.path,
+                tn: meta.tn,
+                title: meta.tn,
+                type: route.type,
+              }
+            );
+          }
+        };
+      });
+
+      await NcHelp.executeOperations(
+        routesInsertion,
+        this.connectionConfig.client
+      );
+    }
+
+    // add new routes
+  }
+
+  protected async getManyToManyRelations(args = {}): Promise<Set<any>> {
+    const metas: Set<any> = await super.getManyToManyRelations(args);
+
+    for (const metaObj of metas) {
+      const ctx = this.generateContextForTable(
+        metaObj.tn,
+        metaObj.columns,
+        [...metaObj.belongsTo, ...metaObj.hasMany],
+        metaObj.hasMany,
+        metaObj.belongsTo
+      );
+
+      const swaggerDoc = await new SwaggerXc({
+        dir: '',
+        ctx: {
+          ...ctx,
+          v: metaObj.v,
+        },
+        filename: '',
+      }).getObject();
+
+      const meta = await this.xcMeta.metaGet(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          title: metaObj.tn,
+          type: 'table',
+        }
+      );
+      const oldSwaggerDoc = JSON.parse(meta.schema);
+
+      // // keep upto 5 schema backup on table update
+      // let previousSchemas = [oldSwaggerDoc]
+      // if (meta.schema_previous) {
+      //   previousSchemas = [...JSON.parse(meta.schema_previous), oldSwaggerDoc].slice(-5);
+      // }
+
+      oldSwaggerDoc.definitions = swaggerDoc.definitions;
+      await this.xcMeta.metaUpdate(
+        this.projectId,
+        this.dbAlias,
+        'nc_models',
+        {
+          schema: JSON.stringify(oldSwaggerDoc),
+          // schema_previous: JSON.stringify(previousSchemas)
+        },
+        {
+          title: metaObj.tn,
+          type: 'table',
+        }
+      );
+    }
+
+    return metas;
+  }
+}

packages/nocodb-nest/src/version-upgrader/v1-legacy/templates/NcTemplateParser.ts

@@ -0,0 +1,253 @@
+import { SqlUiFactory, UITypes } from 'nocodb-sdk';
+import type { MssqlUi, MysqlUi, OracleUi, PgUi, SqliteUi } from 'nocodb-sdk';
+
+export default class NcTemplateParser {
+  sqlUi:
+    | typeof MysqlUi
+    | typeof MssqlUi
+    | typeof PgUi
+    | typeof OracleUi
+    | typeof SqliteUi;
+
+  private _tables: any[];
+  private client: string;
+  private _relations: any[];
+  private _m2mRelations: any[];
+  private _virtualColumns: { [tn: string]: any[] };
+  private prefix: string;
+  private template: any;
+
+  constructor({ client, template, prefix = '' }) {
+    this.client = client;
+    this.sqlUi = SqlUiFactory.create({ client });
+    this.template = template;
+    this.prefix = prefix;
+  }
+
+  public parse(template?: any): any {
+    const tables = [];
+    this.template = template || this.template;
+    const tableTemplates = this.template.tables.map((tableTemplate) => {
+      const t = {
+        ...tableTemplate,
+        tn: this.getTable(tableTemplate.tn),
+        _tn: tableTemplate._tn || tableTemplate.tn,
+      };
+      const table = this.extractTable(t);
+      tables.push(table);
+      return t;
+    });
+
+    this._tables = tables;
+
+    for (const tableTemplate of tableTemplates) {
+      this.extractRelations(tableTemplate);
+      this.extractVirtualColumns(tableTemplate);
+    }
+  }
+
+  private extractTable(tableTemplate) {
+    if (!tableTemplate?.tn) {
+      throw Error('Missing table name in template');
+    }
+
+    const defaultColumns = this.sqlUi
+      .getNewTableColumns()
+      .filter(
+        (column) =>
+          column.cn !== 'title' &&
+          (column.uidt !== 'ID' ||
+            tableTemplate.columns.every((c) => c.uidt !== 'ID'))
+      );
+
+    return {
+      tn: tableTemplate.tn,
+      _tn: tableTemplate._tn,
+      columns: [
+        defaultColumns[0],
+        ...this.extractTableColumns(tableTemplate.columns),
+        ...defaultColumns.slice(1),
+      ],
+    };
+  }
+
+  private extractTableColumns(tableColumns: any[]) {
+    const columns = [];
+    for (const tableColumn of tableColumns) {
+      if (!tableColumn?.cn) {
+        throw Error('Missing column name in template');
+      }
+      switch (tableColumn.uidt) {
+        // case UITypes.ForeignKey:
+        //   // todo :
+        //   this.extractRelations(tableColumn, 'bt');
+        //   break;
+        // case UITypes.LinkToAnotherRecord:
+        //   // todo :
+        //   this.extractRelations(tableColumn, 'hm');
+        //   // this.extractRelations(tableColumn, 'mm');
+        //   break;
+        default:
+          {
+            const colProp = this.sqlUi.getDataTypeForUiType(tableColumn);
+            columns.push({
+              ...this.sqlUi.getNewColumn(''),
+              rqd: false,
+              pk: false,
+              ai: false,
+              cdf: null,
+              un: false,
+              dtx: 'specificType',
+              dtxp: this.sqlUi.getDefaultLengthForDatatype(colProp.dt),
+              dtxs: this.sqlUi.getDefaultScaleForDatatype(colProp.dt),
+              ...colProp,
+              _cn: tableColumn.cn,
+              ...tableColumn,
+            });
+          }
+          break;
+      }
+    }
+    return columns;
+  }
+
+  protected extractRelations(tableTemplate) {
+    if (!this._relations) this._relations = [];
+    if (!this._m2mRelations) this._m2mRelations = [];
+    for (const hasMany of tableTemplate.hasMany || []) {
+      const childTable = this.tables.find(
+        (table) => table.tn === this.getTable(hasMany.tn)
+      );
+      const parentTable = this.tables.find(
+        (table) => table.tn === tableTemplate.tn
+      );
+      const parentPrimaryColumn = parentTable.columns.find(
+        (column) => column.uidt === UITypes.ID
+      );
+      //
+      // // if duplicate relation ignore
+      // if (
+      //   this._relations.some(rl => {
+      //     return (
+      //       (rl.childTable === childTable.tn &&
+      //         rl.parentTable === parentTable.tn) ||
+      //       (rl.parentTable === childTable.tn &&
+      //         rl.childTable === parentTable.tn)
+      //     );
+      //   })
+      // ) {
+      //   continue;f
+      // }
+
+      // add a column in child table
+      const childColumnName = `${tableTemplate.tn}_id`;
+
+      childTable.columns.push({
+        column_name: childColumnName,
+        _cn: childColumnName,
+        rqd: false,
+        pk: false,
+        ai: false,
+        cdf: null,
+        dt: parentPrimaryColumn.dt,
+        dtxp: parentPrimaryColumn.dtxp,
+        dtxs: parentPrimaryColumn.dtxs,
+        un: parentPrimaryColumn.un,
+        altered: 1,
+      });
+
+      // add relation create entry
+      this._relations.push({
+        childColumn: childColumnName,
+        childTable: childTable.tn,
+        onDelete: 'NO ACTION',
+        onUpdate: 'NO ACTION',
+        parentColumn: parentPrimaryColumn.cn,
+        parentTable: tableTemplate.tn,
+        type: this.client === 'sqlite3' ? 'virtual' : 'real',
+        updateRelation: false,
+      });
+    }
+    for (const manyToMany of tableTemplate.manyToMany || []) {
+      // @ts-ignore
+      const childTable = this.tables.find(
+        (table) => table.tn === this.getTable(manyToMany.rtn)
+      );
+      const parentTable = this.tables.find(
+        (table) => table.tn === tableTemplate.tn
+      );
+      const parentPrimaryColumn = parentTable.columns.find(
+        (column) => column.uidt === UITypes.ID
+      );
+      const childPrimaryColumn = childTable.columns.find(
+        (column) => column.uidt === UITypes.ID
+      );
+
+      // if duplicate relation ignore
+      if (
+        this._m2mRelations.some((mm) => {
+          return (
+            (mm.childTable === childTable.tn &&
+              mm.parentTable === parentTable.tn) ||
+            (mm.parentTable === childTable.tn &&
+              mm.childTable === parentTable.tn)
+          );
+        })
+      ) {
+        continue;
+      }
+
+      // add many to many relation create entry
+      this._m2mRelations.push({
+        alias: 'title8',
+        childColumn: childPrimaryColumn.cn,
+        childTable: childTable.tn,
+        onDelete: 'NO ACTION',
+        onUpdate: 'NO ACTION',
+        parentColumn: parentPrimaryColumn.cn,
+        parentTable: parentTable.tn,
+        type: this.client === 'sqlite3' ? 'virtual' : 'real',
+        updateRelation: false,
+      });
+    }
+  }
+
+  private extractVirtualColumns(tableMeta) {
+    if (!this._virtualColumns) this._virtualColumns = {};
+    const virtualColumns = [];
+    for (const v of tableMeta.v || []) {
+      const v1 = { ...v };
+
+      if (v.rl) {
+        v1.rl.rlttn = v1.rl.rltn;
+        v1.rl.rltn = this.getTable(v1.rl.rltn);
+      } else if (v.lk) {
+        v1.lk._ltn = v1.lk.ltn;
+        v1.lk.ltn = this.getTable(v1.lk.ltn);
+      }
+
+      virtualColumns.push(v1);
+    }
+    this.virtualColumns[tableMeta.tn] = virtualColumns;
+  }
+
+  get tables(): any[] {
+    return this._tables;
+  }
+
+  get relations(): any[] {
+    return this._relations;
+  }
+
+  get m2mRelations(): any[] {
+    return this._m2mRelations;
+  }
+
+  get virtualColumns(): { [tn: string]: any[] } {
+    return this._virtualColumns;
+  }
+
+  private getTable(tn) {
+    return `${this.prefix}${tn}`;
+  }
+}