feat: attachment clean up job

packages/nocodb/src/interface/Jobs.ts

@@ -18,6 +18,7 @@ export enum JobTypes {
   CleanUp = 'clean-up',
   DataExport = 'data-export',
   ThumbnailGenerator = 'thumbnail-generator',
+  AttachmentCleanUp = 'attachment-clean-up',
 }
 
 export enum JobStatus {

packages/nocodb/src/models/FileReference.ts

@@ -141,19 +141,6 @@ export default class FileReference {
     );
   }
 
-  public static async hardDelete(
-    context: NcContext,
-    fileReferenceId: string,
-    ncMeta = Noco.ncMeta,
-  ) {
-    await ncMeta.metaDelete(
-      context.workspace_id,
-      context.base_id,
-      MetaTable.FILE_REFERENCES,
-      fileReferenceId,
-    );
-  }
-
   public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) {
     const fileReferenceData = await ncMeta.metaGet2(
       context.workspace_id,

packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@@ -11,6 +11,7 @@ import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export
 import { JobsEventService } from '~/modules/jobs/jobs-event.service';
 import { JobStatus, JobTypes } from '~/interface/Jobs';
 import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor';
+import { AttachmentCleanUpProcessor } from '~/modules/jobs/jobs/attachment-clean-up/attachment-clean-up';
 
 export interface Job {
   id: string;
@@ -37,6 +38,7 @@ export class QueueService {
     protected readonly webhookHandlerProcessor: WebhookHandlerProcessor,
     protected readonly dataExportProcessor: DataExportProcessor,
     protected readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor,
+    protected readonly attachmentCleanUpProcessor: AttachmentCleanUpProcessor,
   ) {
     this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
       const job = this.queueMemory.find((job) => job.id === data.job.id);
@@ -106,6 +108,10 @@ export class QueueService {
       this: this.thumbnailGeneratorProcessor,
       fn: this.thumbnailGeneratorProcessor.job,
     },
+    [JobTypes.AttachmentCleanUp]: {
+      this: this.attachmentCleanUpProcessor,
+      fn: this.attachmentCleanUpProcessor.job,
+    },
   };
 
   async jobWrapper(job: Job) {

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -19,6 +19,7 @@ import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/web
 import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor';
 import { DataExportController } from '~/modules/jobs/jobs/data-export/data-export.controller';
 import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor';
+import { AttachmentCleanUpProcessor } from '~/modules/jobs/jobs/attachment-clean-up/attachment-clean-up';
 
 // Jobs Module Related
 import { JobsLogService } from '~/modules/jobs/jobs/jobs-log.service';
@@ -80,6 +81,7 @@ export const JobsModuleMetadata = {
     WebhookHandlerProcessor,
     DataExportProcessor,
     ThumbnailGeneratorProcessor,
+    AttachmentCleanUpProcessor,
   ],
   exports: ['JobsService'],
 };

packages/nocodb/src/modules/jobs/jobs/attachment-clean-up/attachment-clean-up.ts

@@ -0,0 +1,69 @@
+import path from 'path';
+import debug from 'debug';
+import { Process, Processor } from '@nestjs/bull';
+import { Job } from 'bull';
+import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
+import Noco from '~/Noco';
+import { MetaTable } from '~/utils/globals';
+import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
+
+@Processor(JOBS_QUEUE)
+export class AttachmentCleanUpProcessor {
+  private readonly debugLog = debug('nc:jobs:attachment-clean-up');
+
+  constructor() {}
+
+  @Process(JobTypes.AttachmentCleanUp)
+  async job(job: Job) {
+    this.debugLog(`job started for ${job.id}`);
+
+    const ncMeta = Noco.ncMeta;
+
+    const storageAdapter = await NcPluginMgrv2.storageAdapter();
+
+    const storageAdapterName = storageAdapter.constructor.name;
+
+    const orphanedFilesQueryBuilder = ncMeta
+      .knexConnection(MetaTable.FILE_REFERENCES)
+      .whereIn(
+        'file_url',
+        ncMeta
+          .knexConnection(MetaTable.FILE_REFERENCES)
+          .select('file_url')
+          .groupBy('file_url')
+          .havingRaw('COUNT(*) = COUNT(CASE WHEN deleted THEN 1 END)'),
+      );
+
+    const orphanedFiles = await orphanedFilesQueryBuilder.select('file_url');
+
+    for (const file of orphanedFiles) {
+      let relativePath;
+
+      if (storageAdapterName === 'Local') {
+        relativePath = file.file_url.replace(/^\/?nc\/uploads\//, '');
+      } else {
+        relativePath = new URL(file.file_url).pathname.replace(
+          /^\/?nc\/uploads\//,
+          '',
+        );
+      }
+
+      await storageAdapter.fileDelete(path.join('nc', 'uploads', relativePath));
+
+      const thumbnails = ['tiny.jpg', 'small.jpg', 'card_cover.jpg'];
+
+      for (const thumb of thumbnails) {
+        await storageAdapter.fileDelete(
+          path.join('nc', 'thumbnails', relativePath, thumb),
+        );
+      }
+
+      await ncMeta
+        .knexConnection(MetaTable.FILE_REFERENCES)
+        .where('file_url', file.file_url)
+        .del();
+    }
+
+    this.debugLog(`job completed for ${job.id}`);
+  }
+}