feat: improved migration logic

packages/nocodb/src/modules/jobs/migration-jobs/nc_job_001_attachment.ts

@@ -152,62 +152,10 @@ export class AttachmentMigration {
           .insert(fileReferenceBuffer);
       }
 
-      // eslint-disable-next-line no-constant-condition
-      while (true) {
-        const modelLimit = 100;
-
-        let modelOffset = 0;
-
-        const modelsWithAttachmentColumns = [];
-
-        // get models that have at least one attachment column, and not processed
-
-        // eslint-disable-next-line no-constant-condition
-        while (true) {
-          const selectFields = [
-            ...(Noco.isEE() ? ['fk_workspace_id'] : []),
-            'base_id',
-            'source_id',
-            'fk_model_id',
-          ];
-
-          const models = await ncMeta
-            .knexConnection(MetaTable.COLUMNS)
-            .select(selectFields)
-            .where('uidt', UITypes.Attachment)
-            .whereNotIn(
-              'fk_model_id',
-              ncMeta
-                .knexConnection(temp_processed_models_table)
-                .select('fk_model_id')
-                .where('completed', true),
-            )
-            .groupBy(selectFields)
-            .limit(modelLimit)
-            .offset(modelOffset);
-
-          modelOffset += modelLimit;
-
-          if (!models?.length) {
-            break;
-          }
-
-          modelsWithAttachmentColumns.push(...models);
-        }
-
-        if (!modelsWithAttachmentColumns?.length) {
-          break;
-        }
-
-        this.log(
-          `Found ${modelsWithAttachmentColumns.length} models with attachment columns`,
-        );
-
       let processedModelsCount = 0;
 
-        for (const modelData of modelsWithAttachmentColumns) {
-          const { fk_workspace_id, base_id, source_id, fk_model_id } =
-            modelData;
+      const processModel = async (modelData) => {
+        const { fk_workspace_id, base_id, source_id, fk_model_id } = modelData;
 
         const context = {
           workspace_id: fk_workspace_id,
@@ -218,14 +166,14 @@ export class AttachmentMigration {
 
         if (!source) {
           this.log(`source not found for ${source_id}`);
-            continue;
+          return;
         }
 
         const model = await Model.get(context, fk_model_id);
 
         if (!model) {
           this.log(`model not found for ${fk_model_id}`);
-            continue;
+          return;
         }
 
         await model.getColumns(context);
@@ -238,7 +186,7 @@ export class AttachmentMigration {
 
         if (!dbDriver) {
           this.log(`connection can't achieved for ${source_id}`);
-            continue;
+          return;
         }
 
         const baseModel = await Model.getBaseModelSQL(context, {
@@ -362,10 +310,7 @@ export class AttachmentMigration {
                         // insert file reference if not exists
                         const fileReference = await ncMeta
                           .knexConnection(MetaTable.FILE_REFERENCES)
-                            .where(
-                              'file_url',
-                              attachment.path || attachment.url,
-                            )
+                          .where('file_url', attachment.path || attachment.url)
                           .andWhere('storage', storageAdapterType)
                           .first();
 
@@ -412,8 +357,7 @@ export class AttachmentMigration {
               }
 
               if (updateRequired) {
-                  updateData[column.column_name] =
-                    JSON.stringify(attachmentArr);
+                updateData[column.column_name] = JSON.stringify(attachmentArr);
               }
             }
 
@@ -466,12 +410,77 @@ export class AttachmentMigration {
           .update({ completed: true });
 
         processedModelsCount += 1;
+      };
+
+      const selectFields = [
+        ...(Noco.isEE() ? ['fk_workspace_id'] : []),
+        'base_id',
+        'source_id',
+        'fk_model_id',
+      ];
+
+      const numberOfModelsToBeProcessed = (
+        await ncMeta.knexConnection
+          .from(
+            ncMeta
+              .knexConnection(MetaTable.COLUMNS)
+              .where('uidt', UITypes.Attachment)
+              .whereNotIn(
+                'fk_model_id',
+                ncMeta
+                  .knexConnection(temp_processed_models_table)
+                  .select('fk_model_id')
+                  .where('completed', true),
+              )
+              .groupBy(selectFields)
+              .count('*', { as: 'count' })
+              .as('t'),
+          )
+          .sum('count as count')
+          .first()
+      )?.count;
+
+      const processModelLimit = 100;
+
+      // get models that have at least one attachment column, and not processed
+
+      // eslint-disable-next-line no-constant-condition
+      while (true) {
+        // this will return until all models marked as processed
+        const models = await ncMeta
+          .knexConnection(MetaTable.COLUMNS)
+          .select(selectFields)
+          .where('uidt', UITypes.Attachment)
+          .whereNotIn(
+            'fk_model_id',
+            ncMeta
+              .knexConnection(temp_processed_models_table)
+              .select('fk_model_id')
+              .where('completed', true),
+          )
+          .groupBy(selectFields)
+          .limit(processModelLimit);
+
+        if (!models?.length) {
+          break;
+        }
 
+        for (const model of models) {
+          try {
+            await processModel(model);
             this.log(
-            `Processed ${processedModelsCount} of ${modelsWithAttachmentColumns.length} models`,
+              `Processed ${processedModelsCount} of ${numberOfModelsToBeProcessed} models`,
             );
+          } catch (e) {
+            this.log(`Error processing model ${model.fk_model_id}`);
+            this.log(e);
+          }
         }
       }
+
+      this.log(
+        `Processed total of ${numberOfModelsToBeProcessed} models with attachments`,
+      );
     } catch (e) {
       this.log(`There was an error while processing attachment migration job`);
       this.log(e);