fix: at import links (#8831)

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -15,7 +15,7 @@ import FetchAT from './helpers/fetchAT';
 import { importData } from './helpers/readAndProcessData';
 import EntityMap from './helpers/EntityMap';
 import type { UserType } from 'nocodb-sdk';
-import { type Base, Source } from '~/models';
+import { type Base, Model, Source } from '~/models';
 import { sanitizeColumnName } from '~/helpers';
 import { AttachmentsService } from '~/services/attachments.service';
 import { ColumnsService } from '~/services/columns.service';
@@ -34,6 +34,7 @@ import { FormsService } from '~/services/forms.service';
 import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
 import { GridColumnsService } from '~/services/grid-columns.service';
 import { TelemetryService } from '~/services/telemetry.service';
+import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
 
 const logger = new Logger('at-import');
 
@@ -2521,6 +2522,9 @@ export class AtImportProcessor {
 
           logBasic('Reading Records...');
 
+          const idMap = new Map();
+          const idCounter: Record<string, number> = {};
+
           for (let i = 0; i < ncTblList.list.length; i++) {
             // not a migrated table, skip
             if (
@@ -2552,10 +2556,52 @@ export class AtImportProcessor {
               insertedAssocRef,
               atNcAliasRef,
               ncLinkMappingTable,
+              idMap,
+              idCounter,
               logBasic,
               logDetailed,
               logWarning,
             });
+
+            if (source.type === 'pg') {
+              const baseModel = await Model.getBaseModelSQL(context, {
+                id: ncTblList.list[i].id,
+                viewId: null,
+                dbDriver: await NcConnectionMgrv2.get(source),
+              });
+              await baseModel.dbDriver.raw(
+                `SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
+                [
+                  baseModel.getTnPath(ncTblList.list[i].table_name),
+                  'id',
+                  baseModel.dbDriver.raw(`(SELECT MAX(id) FROM ??)`, [
+                    baseModel.getTnPath(ncTblList.list[i].table_name),
+                  ]),
+                ],
+              );
+            } else if (source.type === 'mssql') {
+              const baseModel = await Model.getBaseModelSQL(context, {
+                id: ncTblList.list[i].id,
+                viewId: null,
+                dbDriver: await NcConnectionMgrv2.get(source),
+              });
+              const res = await baseModel.execAndGetRows(
+                baseModel.dbDriver
+                  .raw(`SELECT MAX(id) as mx FROM ??`, [
+                    baseModel.getTnPath(ncTblList.list[i].table_name),
+                  ])
+                  .toQuery(),
+              );
+
+              await baseModel.dbDriver.raw(
+                `DBCC CHECKIDENT ('??', RESEED, ?)`,
+                [
+                  baseModel.getTnPath(ncTblList.list[i].table_name),
+                  res?.[0]?.mx || 1,
+                ],
+              );
+            }
+
             rtc.data.records += importStats.importedCount;
             rtc.data.nestedLinks += importStats.nestedLinkCount;
 

packages/nocodb/src/modules/jobs/jobs/at-import/helpers/readAndProcessData.ts

@@ -124,6 +124,8 @@ export async function importData(
     insertedAssocRef = {},
     atNcAliasRef,
     ncLinkMappingTable,
+    idMap,
+    idCounter,
   }: {
     baseName: string;
     table: { title?: string; id?: string };
@@ -145,6 +147,8 @@ export async function importData(
     // link related props end
     syncDB;
     services: AirtableImportContext;
+    idMap: Map<string, number>;
+    idCounter: Record<string, number>;
   },
 ): Promise<{
   nestedLinkCount: number;
@@ -188,6 +192,8 @@ export async function importData(
         source,
         services,
         queue,
+        idMap,
+        idCounter,
         logBasic,
         logDetailed,
         logWarning,
@@ -210,12 +216,23 @@ export async function importData(
           () =>
             new Promise(async (resolve) => {
               try {
+                if (idCounter[table.id] === undefined) {
+                  idCounter[table.id] = 1;
+                }
+
                 const { id: rid, ...fields } = record;
+                if (!idMap.has(rid)) {
+                  idMap.set(rid, idCounter[table.id]++);
+                }
                 const r = await nocoBaseDataProcessing_v2(syncDB, table, {
                   id: rid,
                   fields,
                 });
-                tempData.push(r);
+
+                tempData.push({
+                  ...r,
+                  id: idMap.get(rid),
+                });
                 tempCount++;
 
                 if (tempCount >= BULK_DATA_BATCH_COUNT) {
@@ -324,6 +341,8 @@ export async function importLTARData(
     source,
     services,
     queue,
+    idMap,
+    idCounter,
     logBasic = (_str) => {},
     logWarning = (_str) => {},
   }: {
@@ -341,6 +360,8 @@ export async function importLTARData(
     source: Source;
     services: AirtableImportContext;
     queue: PQueue;
+    idMap: Map<string, number>;
+    idCounter: Record<string, number>;
     logBasic: (string) => void;
     logDetailed: (string) => void;
     logWarning: (string) => void;
@@ -412,15 +433,31 @@ export async function importLTARData(
           () =>
             new Promise(async (resolve) => {
               try {
+                if (idCounter[assocMeta.modelMeta.id] === undefined) {
+                  idCounter[assocMeta.modelMeta.id] = 1;
+                }
+
+                if (idCounter[table.id] === undefined) {
+                  idCounter[table.id] = 1;
+                }
+
                 const { id: _atId, ...rec } = record;
 
+                if (!idMap.has(_atId)) {
+                  rec.id = idMap.set(_atId, idCounter[table.id]++);
+                }
+
                 // todo: use actual alias instead of sanitized
                 const links =
                   rec?.[atNcAliasRef[table.id][assocMeta.colMeta.title]] || [];
                 for (const id of links) {
+                  if (!idMap.has(id)) {
+                    idMap.set(id, idCounter[assocMeta.modelMeta.id]++);
+                  }
+
                   assocTableData[assocMeta.modelMeta.id].push({
-                    [assocMeta.curCol.title]: record.id,
-                    [assocMeta.refCol.title]: id,
+                    [assocMeta.curCol.title]: idMap.get(_atId),
+                    [assocMeta.refCol.title]: idMap.get(id),
                   });
 
                   // links can be [] & hence assocTableDta[assocMeta.modelMeta.id] can be [].

packages/nocodb/src/services/columns.service.ts

@@ -8,7 +8,6 @@ import {
   partialUpdateAllowedTypes,
   readonlyMetaAllowedTypes,
   RelationTypes,
-  SourceRestriction,
   substituteColumnAliasWithIdInFormula,
   substituteColumnIdWithAliasInFormula,
   UITypes,