feat: file references & generic data operations

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -45,6 +45,7 @@ import {
   Audit,
   BaseUser,
   Column,
+  FileReference,
   Filter,
   GridViewColumn,
   Model,
@@ -4517,6 +4518,11 @@ class BaseModelSqlv2 {
 
       if (!_trx) await trx.commit();
 
+      await this.clearFileReferences({
+        oldData: data,
+        columns: this.model.columns,
+      });
+
       await this.afterDelete(data, trx, cookie);
       return response;
     } catch (e) {
@@ -4598,8 +4604,6 @@ class BaseModelSqlv2 {
 
       await this.beforeUpdate(data, trx, cookie);
 
-      await this.prepareNocoData(updateObj, false, cookie);
-
       const btForeignKeyColumn = columns.find(
         (c) =>
           c.uidt === UITypes.ForeignKey && data[c.column_name] !== undefined,
@@ -4624,6 +4628,8 @@ class BaseModelSqlv2 {
         NcError.recordNotFound(id);
       }
 
+      await this.prepareNocoData(updateObj, false, cookie, prevData);
+
       const query = this.dbDriver(this.tnPath)
         .update(updateObj)
         .where(await this._wherePk(id, true));
@@ -5456,8 +5462,6 @@ class BaseModelSqlv2 {
           continue;
         }
         if (!raw) {
-          await this.prepareNocoData(d, false, cookie);
-
           pkAndData.push({
             pk: pkValues,
             data: d,
@@ -5478,24 +5482,22 @@ class BaseModelSqlv2 {
               },
             );
 
-            if (oldRecords.length === tempToRead.length) {
+            for (const record of tempToRead) {
-              prevData.push(...oldRecords);
+              const oldRecord = oldRecords.find((r) =>
-            } else {
+                this.comparePks(this._extractPksValues(r), record.pk),
-              for (const recordPk of tempToRead) {
+              );
-                const oldRecord = oldRecords.find((r) =>
-                  this.comparePks(this._extractPksValues(r), recordPk),
-                );
 
-                if (!oldRecord) {
+              if (!oldRecord) {
-                  // throw or skip if no record found
+                // throw or skip if no record found
-                  if (throwExceptionIfNotExist) {
+                if (throwExceptionIfNotExist) {
-                    NcError.recordNotFound(recordPk);
+                  NcError.recordNotFound(record);
-                  }
-                  continue;
                 }
-
+                continue;
-                prevData.push(oldRecord);
               }
+
+              await this.prepareNocoData(record.data, false, cookie, oldRecord);
+
+              prevData.push(oldRecord);
             }
 
             for (const { pk, data } of tempToRead) {
@@ -5809,6 +5811,11 @@ class BaseModelSqlv2 {
 
       await transaction.commit();
 
+      await this.clearFileReferences({
+        oldData: deleted,
+        columns: this.model.columns,
+      });
+
       if (isSingleRecordDeletion) {
         await this.afterDelete(deleted[0], null, cookie);
       } else {
@@ -9136,7 +9143,12 @@ class BaseModelSqlv2 {
     await this.execAndParse(qb, null, { raw: true });
   }
 
-  async prepareNocoData(data, isInsertData = false, cookie?: { user?: any }) {
+  async prepareNocoData(
+    data,
+    isInsertData = false,
+    cookie?: { user?: any },
+    oldData?,
+  ) {
     for (const column of this.model.columns) {
       if (
         ![
@@ -9166,7 +9178,7 @@ class BaseModelSqlv2 {
         }
       }
       if (column.uidt === UITypes.Attachment) {
-        if (data[column.column_name]) {
+        if (data && data[column.column_name]) {
           try {
             if (typeof data[column.column_name] === 'string') {
               data[column.column_name] = JSON.parse(data[column.column_name]);
@@ -9174,31 +9186,113 @@ class BaseModelSqlv2 {
           } catch (e) {
             NcError.invalidAttachmentJson(data[column.column_name]);
           }
+        }
 
-          if (Array.isArray(data[column.column_name])) {
+        if (oldData && oldData[column.column_name]) {
-            const sanitizedAttachments = [];
+          try {
-            for (const attachment of data[column.column_name]) {
+            if (typeof oldData[column.column_name] === 'string') {
-              if (!('url' in attachment) && !('path' in attachment)) {
+              oldData[column.column_name] = JSON.parse(
-                NcError.unprocessableEntity(
+                oldData[column.column_name],
-                  'Attachment object must contain either url or path',
+              );
-                );
+            }
-              }
+          } catch (e) {}
-              sanitizedAttachments.push(
+        }
-                extractProps(attachment, [
+
-                  'url',
+        const regenerateIds = [];
-                  'path',
+
-                  'title',
+        if (!isInsertData) {
-                  'mimetype',
+          const oldAttachmentMap = new Map<
-                  'size',
+            string,
-                  'icon',
+            { url?: string; path?: string }
-                  'width',
+          >(
-                  'height',
+            oldData &&
-                ]),
+            oldData[column.column_name] &&
+            Array.isArray(oldData[column.column_name])
+              ? oldData[column.column_name].map((att) => [att.id, att])
+              : [],
+          );
+
+          const newAttachmentMap = new Map<
+            string,
+            { url?: string; path?: string }
+          >(
+            data[column.column_name] && Array.isArray(data[column.column_name])
+              ? data[column.column_name].map((att) => [att.id, att])
+              : [],
+          );
+
+          for (const [oldId, oldAttachment] of oldAttachmentMap) {
+            if (!newAttachmentMap.has(oldId)) {
+              await FileReference.delete(this.context, oldId);
+            } else if (
+              (oldAttachment.url &&
+                oldAttachment.url !== newAttachmentMap.get(oldId).url) ||
+              (oldAttachment.path &&
+                oldAttachment.path !== newAttachmentMap.get(oldId).path)
+            ) {
+              await FileReference.delete(this.context, oldId);
+              regenerateIds.push(oldId);
+            }
+          }
+
+          for (const [newId, newAttachment] of newAttachmentMap) {
+            if (!oldAttachmentMap.has(newId)) {
+              regenerateIds.push(newId);
+            } else if (
+              (newAttachment.url &&
+                newAttachment.url !== oldAttachmentMap.get(newId).url) ||
+              (newAttachment.path &&
+                newAttachment.path !== oldAttachmentMap.get(newId).path)
+            ) {
+              regenerateIds.push(newId);
+            }
+          }
+        }
+
+        const sanitizedAttachments = [];
+        if (Array.isArray(data[column.column_name])) {
+          for (const attachment of data[column.column_name]) {
+            if (!('url' in attachment) && !('path' in attachment)) {
+              NcError.unprocessableEntity(
+                'Attachment object must contain either url or path',
               );
             }
-            data[column.column_name] = JSON.stringify(sanitizedAttachments);
+            const sanitizedAttachment = extractProps(attachment, [
+              'id',
+              'url',
+              'path',
+              'title',
+              'mimetype',
+              'size',
+              'icon',
+              'width',
+              'height',
+            ]);
+
+            if (
+              isInsertData ||
+              !sanitizedAttachment.id ||
+              regenerateIds.includes(sanitizedAttachment.id)
+            ) {
+              sanitizedAttachment.id = await FileReference.insert(
+                this.context,
+                {
+                  file_url: sanitizedAttachment.url ?? sanitizedAttachment.path,
+                  file_size: sanitizedAttachment.size,
+                  fk_user_id: cookie?.user?.id,
+                  fk_model_id: this.model.id,
+                  fk_column_id: column.id,
+                },
+              );
+            }
+
+            sanitizedAttachments.push(sanitizedAttachment);
           }
         }
+
+        data[column.column_name] = sanitizedAttachments.length
+          ? JSON.stringify(sanitizedAttachments)
+          : null;
       } else if (
         [UITypes.User, UITypes.CreatedBy, UITypes.LastModifiedBy].includes(
           column.uidt,
@@ -9372,6 +9466,50 @@ class BaseModelSqlv2 {
       qb,
     );
   }
+
+  protected async clearFileReferences(args: {
+    oldData?: Record<string, any>[];
+    columns?: Column[];
+  }) {
+    const { oldData, columns } = args;
+
+    const modelColumns = columns || (await this.model.getColumns(this.context));
+
+    const attachmentColumns = modelColumns.filter(
+      (c) => c.uidt === UITypes.Attachment,
+    );
+
+    if (attachmentColumns.length === 0) return;
+
+    for (const column of attachmentColumns) {
+      const oldAttachments = [];
+
+      if (oldData) {
+        for (const row of oldData) {
+          let attachmentRecord = row[column.title];
+          if (attachmentRecord) {
+            try {
+              if (typeof attachmentRecord === 'string') {
+                attachmentRecord = JSON.parse(row[column.title]);
+              }
+              for (const attachment of attachmentRecord) {
+                oldAttachments.push(attachment);
+              }
+            } catch (e) {
+              logger.error(e);
+            }
+          }
+        }
+      }
+
+      if (oldAttachments.length === 0) continue;
+
+      await FileReference.delete(
+        this.context,
+        oldAttachments.filter((at) => at.id).map((at) => at.id),
+      );
+    }
+  }
 }
 
 export function extractSortsObject(

packages/nocodb/src/meta/meta.service.ts

@@ -209,6 +209,64 @@ export class MetaService {
     return insertObj;
   }
 
+  /***
+   * Update multiple records in meta data
+   * @param workspace_id - Workspace id
+   * @param base_id - Base id
+   * @param target - Table name
+   * @param data - Data to be updated
+   * @param ids - Ids of the records to be updated
+   */
+  public async bulkMetaUpdate(
+    workspace_id: string,
+    base_id: string,
+    target: string,
+    data: any | any[],
+    ids: string[],
+  ): Promise<any> {
+    if (Array.isArray(data) ? !data.length : !data) {
+      return [];
+    }
+
+    const query = this.knexConnection(target);
+
+    const at = this.now();
+
+    if (workspace_id === base_id) {
+      if (!Object.values(RootScopes).includes(workspace_id as RootScopes)) {
+        NcError.metaError({
+          message: 'Invalid scope',
+          sql: '',
+        });
+      }
+
+      if (!RootScopeTables[workspace_id].includes(target)) {
+        NcError.metaError({
+          message: 'Table not accessible from this scope',
+          sql: '',
+        });
+      }
+    } else {
+      if (!base_id) {
+        NcError.metaError({
+          message: 'Base ID is required',
+          sql: '',
+        });
+      }
+
+      this.contextCondition(query, workspace_id, base_id, target);
+    }
+
+    const updateObj = {
+      ...data,
+      updated_at: at,
+    };
+
+    query.whereIn('id', ids).update(updateObj);
+
+    return query;
+  }
+
   /***
    * Generate nanoid for the given target
    * @param target - Table name
@@ -252,6 +310,7 @@ export class MetaService {
       [MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE]: 'cnp',
       [MetaTable.JOBS]: 'job',
       [MetaTable.INTEGRATIONS]: 'int',
+      [MetaTable.FILE_REFERENCES]: 'at',
     };
 
     const prefix = prefixMap[target] || 'nc';

packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts

@@ -43,6 +43,7 @@ import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs';
 import * as nc_054_id_length from '~/meta/migrations/v2/nc_054_id_length';
 import * as nc_055_junction_pk from '~/meta/migrations/v2/nc_055_junction_pk';
 import * as nc_056_integration from '~/meta/migrations/v2/nc_056_integration';
+import * as nc_057_file_references from '~/meta/migrations/v2/nc_057_file_references';
 
 // Create a custom migration source class
 export default class XcMigrationSourcev2 {
@@ -97,6 +98,7 @@ export default class XcMigrationSourcev2 {
       'nc_054_id_length',
       'nc_055_junction_pk',
       'nc_056_integration',
+      'nc_057_file_references',
     ]);
   }
 
@@ -196,6 +198,8 @@ export default class XcMigrationSourcev2 {
         return nc_055_junction_pk;
       case 'nc_056_integration':
         return nc_056_integration;
+      case 'nc_057_file_references':
+        return nc_057_file_references;
     }
   }
 }

packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts

@@ -0,0 +1,26 @@
+import type { Knex } from 'knex';
+import { MetaTable } from '~/utils/globals';
+
+const up = async (knex: Knex) => {
+  await knex.schema.createTable(MetaTable.FILE_REFERENCES, (table) => {
+    table.string('id', 20).primary().notNullable();
+    table.string('storage');
+    table.text('file_url');
+    table.integer('file_size');
+    table.string('fk_user_id', 20);
+    table.string('fk_workspace_id', 20);
+    table.string('base_id', 20);
+    table.string('fk_model_id', 20);
+    table.string('fk_column_id', 20);
+    table.boolean('deleted').defaultTo(false);
+    table.timestamps(true, true);
+
+    // TODO: Add indexes
+  });
+};
+
+const down = async (knex: Knex) => {
+  await knex.schema.dropTable(MetaTable.FILE_REFERENCES);
+};
+
+export { up, down };

packages/nocodb/src/models/FileReference.ts

@@ -0,0 +1,147 @@
+import type { NcContext } from '~/interface/config';
+import Noco from '~/Noco';
+import { MetaTable } from '~/utils/globals';
+import { extractProps } from '~/helpers/extractProps';
+
+export default class FileReference {
+  id: string;
+  storage: string;
+  file_url: string;
+  file_size: number;
+  fk_user_id: string;
+  fk_workspace_id: string;
+  base_id: string;
+  fk_model_id: string;
+  fk_column_id: string;
+  deleted: boolean;
+  created_at: Date;
+  updated_at: Date;
+
+  constructor(data: Partial<FileReference>) {
+    Object.assign(this, data);
+  }
+
+  public static async insert(
+    context: NcContext,
+    fileRefObj: Partial<FileReference>,
+    ncMeta = Noco.ncMeta,
+  ) {
+    const insertObj = extractProps(fileRefObj, [
+      'storage',
+      'file_url',
+      'file_size',
+      'fk_user_id',
+      'fk_model_id',
+      'fk_column_id',
+      'deleted',
+    ]);
+
+    const { id } = await ncMeta.metaInsert2(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      insertObj,
+    );
+
+    return id;
+  }
+
+  public static async bulkInsert(
+    context: NcContext,
+    fileRefObjs: Partial<FileReference>[],
+    ncMeta = Noco.ncMeta,
+  ) {
+    let insertObjs = fileRefObjs.map((fileRefObj) =>
+      extractProps(fileRefObj, [
+        'storage',
+        'file_url',
+        'file_size',
+        'fk_user_id',
+        'fk_model_id',
+        'fk_column_id',
+        'deleted',
+      ]),
+    );
+
+    // insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES);
+    // use promise.all to populate the ids
+    insertObjs = await Promise.all(
+      insertObjs.map(async (insertObj) => {
+        insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES);
+        return insertObj;
+      }),
+    );
+
+    await ncMeta.bulkMetaInsert(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      insertObjs,
+    );
+
+    return insertObjs;
+  }
+
+  public static async update(
+    context: NcContext,
+    fileReferenceId: string | string[],
+    fileReferenceObj: Partial<FileReference>,
+    ncMeta = Noco.ncMeta,
+  ) {
+    const updateObj = extractProps(fileReferenceObj, ['deleted']);
+
+    fileReferenceId = Array.isArray(fileReferenceId)
+      ? fileReferenceId
+      : [fileReferenceId];
+
+    return ncMeta.bulkMetaUpdate(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      updateObj,
+      fileReferenceId,
+    );
+  }
+
+  public static async delete(
+    context: NcContext,
+    fileReferenceId: string | string[],
+    ncMeta = Noco.ncMeta,
+  ) {
+    fileReferenceId = Array.isArray(fileReferenceId)
+      ? fileReferenceId
+      : [fileReferenceId];
+
+    await ncMeta.bulkMetaUpdate(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      { deleted: true },
+      fileReferenceId,
+    );
+  }
+
+  public static async hardDelete(
+    context: NcContext,
+    fileReferenceId: string,
+    ncMeta = Noco.ncMeta,
+  ) {
+    await ncMeta.metaDelete(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      fileReferenceId,
+    );
+  }
+
+  public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) {
+    const fileReferenceData = await ncMeta.metaGet2(
+      context.workspace_id,
+      context.base_id,
+      MetaTable.FILE_REFERENCES,
+      id,
+    );
+
+    return fileReferenceData && new FileReference(fileReferenceData);
+  }
+}

packages/nocodb/src/models/index.ts

@@ -45,3 +45,4 @@ export { default as Extension } from './Extension';
 export { default as Comment } from './Comment';
 export { default as Job } from './Job';
 export { default as Integration } from './Integration';
+export { default as FileReference } from './FileReference';

packages/nocodb/src/services/attachments.service.ts

@@ -14,7 +14,7 @@ import type Sharp from 'sharp';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
 import mimetypes, { mimeIcons } from '~/utils/mimeTypes';
-import { PresignedUrl } from '~/models';
+import { FileReference, PresignedUrl } from '~/models';
 import { utf8ify } from '~/helpers/stringHelpers';
 import { NcError } from '~/helpers/catchError';
 import { IJobsService } from '~/modules/jobs/jobs-service.interface';
@@ -114,6 +114,21 @@ export class AttachmentsService {
             file,
           );
 
+          await FileReference.insert(
+            {
+              workspace_id: RootScopes.ROOT,
+              base_id: RootScopes.ROOT,
+            },
+            {
+              storage: storageAdapter.constructor.name,
+              file_url:
+                url ?? path.join('download', filePath.join('/'), fileName),
+              file_size: file.size,
+              fk_user_id: userId,
+              deleted: true, // root file references are always deleted as they are not associated with any record
+            },
+          );
+
           const attachment: AttachmentObject = {
             ...(url
               ? { url }
@@ -228,6 +243,22 @@ export class AttachmentsService {
               },
             );
 
+          await FileReference.insert(
+            {
+              workspace_id: RootScopes.ROOT,
+              base_id: RootScopes.ROOT,
+            },
+            {
+              storage: storageAdapter.constructor.name,
+              file_url:
+                attachmentUrl ??
+                path.join('download', filePath.join('/'), fileName),
+              file_size: file.size,
+              fk_user_id: userId,
+              deleted: true, // root file references are always deleted as they are not associated with any record
+            },
+          );
+
           const tempMetadata: {
             width?: number;
             height?: number;

packages/nocodb/src/utils/globals.ts

@@ -53,6 +53,7 @@ export enum MetaTable {
   COMMENTS_REACTIONS = 'nc_comment_reactions',
   JOBS = 'nc_jobs',
   INTEGRATIONS = 'nc_integrations_v2',
+  FILE_REFERENCES = 'nc_file_references',
 }
 
 export enum MetaTableOldV2 {
@@ -286,6 +287,7 @@ export const RootScopeTables = {
     MetaTable.STORE,
     MetaTable.NOTIFICATION,
     MetaTable.JOBS,
+    MetaTable.FILE_REFERENCES,
     // Temporarily added need to be discussed within team
     MetaTable.AUDIT,
   ],