feat: generate thumbnail only if missing

packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts

@@ -19,23 +19,19 @@ export class ThumbnailGeneratorProcessor {
 
   @Process(JobTypes.ThumbnailGenerator)
   async job(job: Job<ThumbnailGeneratorJobData>) {
-    try {
-      const { attachments } = job.data;
-
-      const thumbnailPromises = attachments.map(async (attachment) => {
-        const thumbnail = await this.generateThumbnail(attachment);
-        return {
-          path: attachment.path ?? attachment.url,
-          card_cover: thumbnail?.card_cover,
-          small: thumbnail?.small,
-          tiny: thumbnail?.tiny,
-        };
-      });
-
-      return await Promise.all(thumbnailPromises);
-    } catch (error) {
-      this.logger.error('Failed to generate thumbnails', error.stack as string);
-    }
+    const { attachments } = job.data;
+
+    const thumbnailPromises = attachments.map(async (attachment) => {
+      const thumbnail = await this.generateThumbnail(attachment);
+      return {
+        path: attachment.path ?? attachment.url,
+        card_cover: thumbnail?.card_cover,
+        small: thumbnail?.small,
+        tiny: thumbnail?.tiny,
+      };
+    });
+
+    return await Promise.all(thumbnailPromises);
   }
 
   private async generateThumbnail(

packages/nocodb/src/modules/jobs/migration-jobs/nc_job_002_thumbnail.ts

@@ -8,6 +8,7 @@ import Noco from '~/Noco';
 import mimetypes from '~/utils/mimeTypes';
 import { RootScopes } from '~/utils/globals';
 import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor';
+import { getPathFromUrl } from '~/helpers/attachmentHelpers';
 
 @Injectable()
 export class ThumbnailMigration {
@@ -26,11 +27,14 @@ export class ThumbnailMigration {
     try {
       const ncMeta = Noco.ncMeta;
 
+      const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta);
+
       const temp_file_references_table = 'nc_temp_file_references';
 
       const fileReferencesTableExists =
         await ncMeta.knexConnection.schema.hasTable(temp_file_references_table);
 
+      // fallback scanning all files if temp table is not generated from previous migration
       if (!fileReferencesTableExists) {
         // create temp file references table if not exists
         await ncMeta.knexConnection.schema.createTable(
@@ -46,9 +50,6 @@ export class ThumbnailMigration {
           },
         );
 
-        // fallback scanning all files if temp table is not generated from previous migration
-        const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta);
-
         const fileScanStream = await storageAdapter.scanFiles('nc/uploads/**');
 
         const fileReferenceBuffer = [];
@@ -160,6 +161,49 @@ export class ThumbnailMigration {
         }
 
         try {
+          // check if thumbnails exist
+          for (const fileReference of fileReferences) {
+            let relativePath;
+
+            const isUrl = /^https?:\/\//i.test(fileReference.file_path);
+            if (isUrl) {
+              relativePath = getPathFromUrl(fileReference.file_path).replace(
+                /^\/+/,
+                '',
+              );
+            } else {
+              relativePath = fileReference.file_path;
+            }
+
+            const thumbnailRoot = relativePath.replace(
+              /nc\/uploads/,
+              'nc/thumbnails',
+            );
+
+            try {
+              const thumbnails = await storageAdapter.getDirectoryList(
+                thumbnailRoot,
+              );
+
+              if (
+                ['card_cover.jpg', 'small.jpg', 'tiny.jpg'].every((t) =>
+                  thumbnails.includes(t),
+                )
+              ) {
+                await ncMeta
+                  .knexConnection(temp_file_references_table)
+                  .where('file_path', fileReference.file_path)
+                  .update({
+                    thumbnail_generated: true,
+                  });
+
+                fileReference.thumbnail_generated = true;
+              }
+            } catch (e) {
+              // ignore error
+            }
+          }
+
           // manually call thumbnail generator job to control the concurrency
           await this.thumbnailGeneratorProcessor.job({
             data: {
@@ -167,20 +211,25 @@ export class ThumbnailMigration {
                 base_id: RootScopes.ROOT,
                 workspace_id: RootScopes.ROOT,
               },
-              attachments: fileReferences.map((f) => {
-                const isUrl = /^https?:\/\//i.test(f.file_path);
-                if (isUrl) {
-                  return {
-                    url: f.file_path,
-                    mimetype: f.mimetype,
-                  };
-                } else {
-                  return {
-                    path: path.join('download', f.file_path),
-                    mimetype: f.mimetype,
-                  };
-                }
-              }),
+              attachments: fileReferences
+                .filter((f) => !f.thumbnail_generated)
+                .map((f) => {
+                  const isUrl = /^https?:\/\//i.test(f.file_path);
+                  if (isUrl) {
+                    return {
+                      url: f.file_path,
+                      mimetype: f.mimetype,
+                    };
+                  } else {
+                    return {
+                      path: path.join(
+                        'download',
+                        f.file_path.replace(/^nc\/uploads\//, ''),
+                      ),
+                      mimetype: f.mimetype,
+                    };
+                  }
+                }),
             },
           } as any);
 

packages/nocodb/src/plugins/GenericS3/GenericS3.ts

@@ -1,6 +1,7 @@
 import fs from 'fs';
 import { promisify } from 'util';
 import { Readable } from 'stream';
+import path from 'path';
 import axios from 'axios';
 import { useAgent } from 'request-filtering-agent';
 import {
@@ -193,9 +194,17 @@ export default class GenericS3 implements IStorageAdapterV2 {
     return Promise.resolve(undefined);
   }
 
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
+  public async getDirectoryList(prefix: string): Promise<string[]> {
+    return this.s3Client
+      .listObjectsV2({
+        Prefix: prefix,
+        Bucket: this.input.bucket,
+      })
+      .then((response) => {
+        return response.Contents.map((content) => {
+          return path.basename(content.Key);
+        });
+      });
   }
 
   public async fileDelete(key: string): Promise<any> {