From 256f5c1b5b09ea305adb591d8d98943d08baaec7 Mon Sep 17 00:00:00 2001 From: mertmit Date: Sat, 10 Aug 2024 07:31:54 +0000 Subject: [PATCH] feat: file references & generic data operations --- packages/nocodb/src/db/BaseModelSqlv2.ts | 220 ++++++++++++++---- packages/nocodb/src/meta/meta.service.ts | 59 +++++ .../meta/migrations/XcMigrationSourcev2.ts | 4 + .../migrations/v2/nc_057_file_references.ts | 26 +++ packages/nocodb/src/models/FileReference.ts | 147 ++++++++++++ packages/nocodb/src/models/index.ts | 1 + .../src/services/attachments.service.ts | 33 ++- packages/nocodb/src/utils/globals.ts | 2 + 8 files changed, 450 insertions(+), 42 deletions(-) create mode 100644 packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts create mode 100644 packages/nocodb/src/models/FileReference.ts diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index aa583b26c7..dee6a293fe 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -45,6 +45,7 @@ import { Audit, BaseUser, Column, + FileReference, Filter, GridViewColumn, Model, @@ -4517,6 +4518,11 @@ class BaseModelSqlv2 { if (!_trx) await trx.commit(); + await this.clearFileReferences({ + oldData: data, + columns: this.model.columns, + }); + await this.afterDelete(data, trx, cookie); return response; } catch (e) { @@ -4598,8 +4604,6 @@ class BaseModelSqlv2 { await this.beforeUpdate(data, trx, cookie); - await this.prepareNocoData(updateObj, false, cookie); - const btForeignKeyColumn = columns.find( (c) => c.uidt === UITypes.ForeignKey && data[c.column_name] !== undefined, @@ -4624,6 +4628,8 @@ class BaseModelSqlv2 { NcError.recordNotFound(id); } + await this.prepareNocoData(updateObj, false, cookie, prevData); + const query = this.dbDriver(this.tnPath) .update(updateObj) .where(await this._wherePk(id, true)); @@ -5456,8 +5462,6 @@ class BaseModelSqlv2 { continue; } if (!raw) { - await this.prepareNocoData(d, false, cookie); - pkAndData.push({ pk: pkValues, data: d, @@ -5478,24 +5482,22 @@ class BaseModelSqlv2 { }, ); - if (oldRecords.length === tempToRead.length) { - prevData.push(...oldRecords); - } else { - for (const recordPk of tempToRead) { - const oldRecord = oldRecords.find((r) => - this.comparePks(this._extractPksValues(r), recordPk), - ); + for (const record of tempToRead) { + const oldRecord = oldRecords.find((r) => + this.comparePks(this._extractPksValues(r), record.pk), + ); - if (!oldRecord) { - // throw or skip if no record found - if (throwExceptionIfNotExist) { - NcError.recordNotFound(recordPk); - } - continue; + if (!oldRecord) { + // throw or skip if no record found + if (throwExceptionIfNotExist) { + NcError.recordNotFound(record); } - - prevData.push(oldRecord); + continue; } + + await this.prepareNocoData(record.data, false, cookie, oldRecord); + + prevData.push(oldRecord); } for (const { pk, data } of tempToRead) { @@ -5809,6 +5811,11 @@ class BaseModelSqlv2 { await transaction.commit(); + await this.clearFileReferences({ + oldData: deleted, + columns: this.model.columns, + }); + if (isSingleRecordDeletion) { await this.afterDelete(deleted[0], null, cookie); } else { @@ -9136,7 +9143,12 @@ class BaseModelSqlv2 { await this.execAndParse(qb, null, { raw: true }); } - async prepareNocoData(data, isInsertData = false, cookie?: { user?: any }) { + async prepareNocoData( + data, + isInsertData = false, + cookie?: { user?: any }, + oldData?, + ) { for (const column of this.model.columns) { if ( ![ @@ -9166,7 +9178,7 @@ class BaseModelSqlv2 { } } if (column.uidt === UITypes.Attachment) { - if (data[column.column_name]) { + if (data && data[column.column_name]) { try { if (typeof data[column.column_name] === 'string') { data[column.column_name] = JSON.parse(data[column.column_name]); @@ -9174,31 +9186,113 @@ class BaseModelSqlv2 { } catch (e) { NcError.invalidAttachmentJson(data[column.column_name]); } + } - if (Array.isArray(data[column.column_name])) { - const sanitizedAttachments = []; - for (const attachment of data[column.column_name]) { - if (!('url' in attachment) && !('path' in attachment)) { - NcError.unprocessableEntity( - 'Attachment object must contain either url or path', - ); - } - sanitizedAttachments.push( - extractProps(attachment, [ - 'url', - 'path', - 'title', - 'mimetype', - 'size', - 'icon', - 'width', - 'height', - ]), + if (oldData && oldData[column.column_name]) { + try { + if (typeof oldData[column.column_name] === 'string') { + oldData[column.column_name] = JSON.parse( + oldData[column.column_name], + ); + } + } catch (e) {} + } + + const regenerateIds = []; + + if (!isInsertData) { + const oldAttachmentMap = new Map< + string, + { url?: string; path?: string } + >( + oldData && + oldData[column.column_name] && + Array.isArray(oldData[column.column_name]) + ? oldData[column.column_name].map((att) => [att.id, att]) + : [], + ); + + const newAttachmentMap = new Map< + string, + { url?: string; path?: string } + >( + data[column.column_name] && Array.isArray(data[column.column_name]) + ? data[column.column_name].map((att) => [att.id, att]) + : [], + ); + + for (const [oldId, oldAttachment] of oldAttachmentMap) { + if (!newAttachmentMap.has(oldId)) { + await FileReference.delete(this.context, oldId); + } else if ( + (oldAttachment.url && + oldAttachment.url !== newAttachmentMap.get(oldId).url) || + (oldAttachment.path && + oldAttachment.path !== newAttachmentMap.get(oldId).path) + ) { + await FileReference.delete(this.context, oldId); + regenerateIds.push(oldId); + } + } + + for (const [newId, newAttachment] of newAttachmentMap) { + if (!oldAttachmentMap.has(newId)) { + regenerateIds.push(newId); + } else if ( + (newAttachment.url && + newAttachment.url !== oldAttachmentMap.get(newId).url) || + (newAttachment.path && + newAttachment.path !== oldAttachmentMap.get(newId).path) + ) { + regenerateIds.push(newId); + } + } + } + + const sanitizedAttachments = []; + if (Array.isArray(data[column.column_name])) { + for (const attachment of data[column.column_name]) { + if (!('url' in attachment) && !('path' in attachment)) { + NcError.unprocessableEntity( + 'Attachment object must contain either url or path', ); } - data[column.column_name] = JSON.stringify(sanitizedAttachments); + const sanitizedAttachment = extractProps(attachment, [ + 'id', + 'url', + 'path', + 'title', + 'mimetype', + 'size', + 'icon', + 'width', + 'height', + ]); + + if ( + isInsertData || + !sanitizedAttachment.id || + regenerateIds.includes(sanitizedAttachment.id) + ) { + sanitizedAttachment.id = await FileReference.insert( + this.context, + { + file_url: sanitizedAttachment.url ?? sanitizedAttachment.path, + file_size: sanitizedAttachment.size, + fk_user_id: cookie?.user?.id, + fk_model_id: this.model.id, + fk_column_id: column.id, + }, + ); + } + + sanitizedAttachments.push(sanitizedAttachment); } } + + data[column.column_name] = sanitizedAttachments.length + ? JSON.stringify(sanitizedAttachments) + : null; } else if ( [UITypes.User, UITypes.CreatedBy, UITypes.LastModifiedBy].includes( column.uidt, @@ -9372,6 +9466,50 @@ class BaseModelSqlv2 { qb, ); } + + protected async clearFileReferences(args: { + oldData?: Record[]; + columns?: Column[]; + }) { + const { oldData, columns } = args; + + const modelColumns = columns || (await this.model.getColumns(this.context)); + + const attachmentColumns = modelColumns.filter( + (c) => c.uidt === UITypes.Attachment, + ); + + if (attachmentColumns.length === 0) return; + + for (const column of attachmentColumns) { + const oldAttachments = []; + + if (oldData) { + for (const row of oldData) { + let attachmentRecord = row[column.title]; + if (attachmentRecord) { + try { + if (typeof attachmentRecord === 'string') { + attachmentRecord = JSON.parse(row[column.title]); + } + for (const attachment of attachmentRecord) { + oldAttachments.push(attachment); + } + } catch (e) { + logger.error(e); + } + } + } + } + + if (oldAttachments.length === 0) continue; + + await FileReference.delete( + this.context, + oldAttachments.filter((at) => at.id).map((at) => at.id), + ); + } + } } export function extractSortsObject( diff --git a/packages/nocodb/src/meta/meta.service.ts b/packages/nocodb/src/meta/meta.service.ts index 0c4b666fa1..a8b75d389d 100644 --- a/packages/nocodb/src/meta/meta.service.ts +++ b/packages/nocodb/src/meta/meta.service.ts @@ -209,6 +209,64 @@ export class MetaService { return insertObj; } + /*** + * Update multiple records in meta data + * @param workspace_id - Workspace id + * @param base_id - Base id + * @param target - Table name + * @param data - Data to be updated + * @param ids - Ids of the records to be updated + */ + public async bulkMetaUpdate( + workspace_id: string, + base_id: string, + target: string, + data: any | any[], + ids: string[], + ): Promise { + if (Array.isArray(data) ? !data.length : !data) { + return []; + } + + const query = this.knexConnection(target); + + const at = this.now(); + + if (workspace_id === base_id) { + if (!Object.values(RootScopes).includes(workspace_id as RootScopes)) { + NcError.metaError({ + message: 'Invalid scope', + sql: '', + }); + } + + if (!RootScopeTables[workspace_id].includes(target)) { + NcError.metaError({ + message: 'Table not accessible from this scope', + sql: '', + }); + } + } else { + if (!base_id) { + NcError.metaError({ + message: 'Base ID is required', + sql: '', + }); + } + + this.contextCondition(query, workspace_id, base_id, target); + } + + const updateObj = { + ...data, + updated_at: at, + }; + + query.whereIn('id', ids).update(updateObj); + + return query; + } + /*** * Generate nanoid for the given target * @param target - Table name @@ -252,6 +310,7 @@ export class MetaService { [MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE]: 'cnp', [MetaTable.JOBS]: 'job', [MetaTable.INTEGRATIONS]: 'int', + [MetaTable.FILE_REFERENCES]: 'at', }; const prefix = prefixMap[target] || 'nc'; diff --git a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts index 04f105947f..e9f613555b 100644 --- a/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts +++ b/packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts @@ -43,6 +43,7 @@ import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs'; import * as nc_054_id_length from '~/meta/migrations/v2/nc_054_id_length'; import * as nc_055_junction_pk from '~/meta/migrations/v2/nc_055_junction_pk'; import * as nc_056_integration from '~/meta/migrations/v2/nc_056_integration'; +import * as nc_057_file_references from '~/meta/migrations/v2/nc_057_file_references'; // Create a custom migration source class export default class XcMigrationSourcev2 { @@ -97,6 +98,7 @@ export default class XcMigrationSourcev2 { 'nc_054_id_length', 'nc_055_junction_pk', 'nc_056_integration', + 'nc_057_file_references', ]); } @@ -196,6 +198,8 @@ export default class XcMigrationSourcev2 { return nc_055_junction_pk; case 'nc_056_integration': return nc_056_integration; + case 'nc_057_file_references': + return nc_057_file_references; } } } diff --git a/packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts b/packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts new file mode 100644 index 0000000000..2cc6037c5b --- /dev/null +++ b/packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts @@ -0,0 +1,26 @@ +import type { Knex } from 'knex'; +import { MetaTable } from '~/utils/globals'; + +const up = async (knex: Knex) => { + await knex.schema.createTable(MetaTable.FILE_REFERENCES, (table) => { + table.string('id', 20).primary().notNullable(); + table.string('storage'); + table.text('file_url'); + table.integer('file_size'); + table.string('fk_user_id', 20); + table.string('fk_workspace_id', 20); + table.string('base_id', 20); + table.string('fk_model_id', 20); + table.string('fk_column_id', 20); + table.boolean('deleted').defaultTo(false); + table.timestamps(true, true); + + // TODO: Add indexes + }); +}; + +const down = async (knex: Knex) => { + await knex.schema.dropTable(MetaTable.FILE_REFERENCES); +}; + +export { up, down }; diff --git a/packages/nocodb/src/models/FileReference.ts b/packages/nocodb/src/models/FileReference.ts new file mode 100644 index 0000000000..5e79e506f8 --- /dev/null +++ b/packages/nocodb/src/models/FileReference.ts @@ -0,0 +1,147 @@ +import type { NcContext } from '~/interface/config'; +import Noco from '~/Noco'; +import { MetaTable } from '~/utils/globals'; +import { extractProps } from '~/helpers/extractProps'; + +export default class FileReference { + id: string; + storage: string; + file_url: string; + file_size: number; + fk_user_id: string; + fk_workspace_id: string; + base_id: string; + fk_model_id: string; + fk_column_id: string; + deleted: boolean; + created_at: Date; + updated_at: Date; + + constructor(data: Partial) { + Object.assign(this, data); + } + + public static async insert( + context: NcContext, + fileRefObj: Partial, + ncMeta = Noco.ncMeta, + ) { + const insertObj = extractProps(fileRefObj, [ + 'storage', + 'file_url', + 'file_size', + 'fk_user_id', + 'fk_model_id', + 'fk_column_id', + 'deleted', + ]); + + const { id } = await ncMeta.metaInsert2( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + insertObj, + ); + + return id; + } + + public static async bulkInsert( + context: NcContext, + fileRefObjs: Partial[], + ncMeta = Noco.ncMeta, + ) { + let insertObjs = fileRefObjs.map((fileRefObj) => + extractProps(fileRefObj, [ + 'storage', + 'file_url', + 'file_size', + 'fk_user_id', + 'fk_model_id', + 'fk_column_id', + 'deleted', + ]), + ); + + // insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES); + // use promise.all to populate the ids + insertObjs = await Promise.all( + insertObjs.map(async (insertObj) => { + insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES); + return insertObj; + }), + ); + + await ncMeta.bulkMetaInsert( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + insertObjs, + ); + + return insertObjs; + } + + public static async update( + context: NcContext, + fileReferenceId: string | string[], + fileReferenceObj: Partial, + ncMeta = Noco.ncMeta, + ) { + const updateObj = extractProps(fileReferenceObj, ['deleted']); + + fileReferenceId = Array.isArray(fileReferenceId) + ? fileReferenceId + : [fileReferenceId]; + + return ncMeta.bulkMetaUpdate( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + updateObj, + fileReferenceId, + ); + } + + public static async delete( + context: NcContext, + fileReferenceId: string | string[], + ncMeta = Noco.ncMeta, + ) { + fileReferenceId = Array.isArray(fileReferenceId) + ? fileReferenceId + : [fileReferenceId]; + + await ncMeta.bulkMetaUpdate( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + { deleted: true }, + fileReferenceId, + ); + } + + public static async hardDelete( + context: NcContext, + fileReferenceId: string, + ncMeta = Noco.ncMeta, + ) { + await ncMeta.metaDelete( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + fileReferenceId, + ); + } + + public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) { + const fileReferenceData = await ncMeta.metaGet2( + context.workspace_id, + context.base_id, + MetaTable.FILE_REFERENCES, + id, + ); + + return fileReferenceData && new FileReference(fileReferenceData); + } +} diff --git a/packages/nocodb/src/models/index.ts b/packages/nocodb/src/models/index.ts index 489a1f0bf4..ef030b435d 100644 --- a/packages/nocodb/src/models/index.ts +++ b/packages/nocodb/src/models/index.ts @@ -45,3 +45,4 @@ export { default as Extension } from './Extension'; export { default as Comment } from './Comment'; export { default as Job } from './Job'; export { default as Integration } from './Integration'; +export { default as FileReference } from './FileReference'; diff --git a/packages/nocodb/src/services/attachments.service.ts b/packages/nocodb/src/services/attachments.service.ts index 48968958f8..be6bbea002 100644 --- a/packages/nocodb/src/services/attachments.service.ts +++ b/packages/nocodb/src/services/attachments.service.ts @@ -14,7 +14,7 @@ import type Sharp from 'sharp'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; import mimetypes, { mimeIcons } from '~/utils/mimeTypes'; -import { PresignedUrl } from '~/models'; +import { FileReference, PresignedUrl } from '~/models'; import { utf8ify } from '~/helpers/stringHelpers'; import { NcError } from '~/helpers/catchError'; import { IJobsService } from '~/modules/jobs/jobs-service.interface'; @@ -114,6 +114,21 @@ export class AttachmentsService { file, ); + await FileReference.insert( + { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + }, + { + storage: storageAdapter.constructor.name, + file_url: + url ?? path.join('download', filePath.join('/'), fileName), + file_size: file.size, + fk_user_id: userId, + deleted: true, // root file references are always deleted as they are not associated with any record + }, + ); + const attachment: AttachmentObject = { ...(url ? { url } @@ -228,6 +243,22 @@ export class AttachmentsService { }, ); + await FileReference.insert( + { + workspace_id: RootScopes.ROOT, + base_id: RootScopes.ROOT, + }, + { + storage: storageAdapter.constructor.name, + file_url: + attachmentUrl ?? + path.join('download', filePath.join('/'), fileName), + file_size: file.size, + fk_user_id: userId, + deleted: true, // root file references are always deleted as they are not associated with any record + }, + ); + const tempMetadata: { width?: number; height?: number; diff --git a/packages/nocodb/src/utils/globals.ts b/packages/nocodb/src/utils/globals.ts index 1d46aaa37d..a2705597ee 100644 --- a/packages/nocodb/src/utils/globals.ts +++ b/packages/nocodb/src/utils/globals.ts @@ -53,6 +53,7 @@ export enum MetaTable { COMMENTS_REACTIONS = 'nc_comment_reactions', JOBS = 'nc_jobs', INTEGRATIONS = 'nc_integrations_v2', + FILE_REFERENCES = 'nc_file_references', } export enum MetaTableOldV2 { @@ -286,6 +287,7 @@ export const RootScopeTables = { MetaTable.STORE, MetaTable.NOTIFICATION, MetaTable.JOBS, + MetaTable.FILE_REFERENCES, // Temporarily added need to be discussed within team MetaTable.AUDIT, ],