github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: file references & generic data operations

nc-feat/attachment-clean-up
mertmit 4 months ago
parent
4020b11c47
commit
256f5c1b5b
8 changed files with 450 additions and 42 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 220
      packages/nocodb/src/db/BaseModelSqlv2.ts
  2. 59
      packages/nocodb/src/meta/meta.service.ts
  3. 4
      packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
  4. 26
      packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts
  5. 147
      packages/nocodb/src/models/FileReference.ts
  6. 1
      packages/nocodb/src/models/index.ts
  7. 33
      packages/nocodb/src/services/attachments.service.ts
  8. 2
      packages/nocodb/src/utils/globals.ts

220
packages/nocodb/src/db/BaseModelSqlv2.ts
Unescape View File

@ -45,6 +45,7 @@ import {
Audit,
BaseUser,
Column,
FileReference,
Filter,
GridViewColumn,
Model,
@ -4517,6 +4518,11 @@ class BaseModelSqlv2 {
if (!_trx) await trx.commit();
await this.clearFileReferences({
oldData: data,
columns: this.model.columns,
});
await this.afterDelete(data, trx, cookie);
return response;
} catch (e) {
@ -4598,8 +4604,6 @@ class BaseModelSqlv2 {
await this.beforeUpdate(data, trx, cookie);
await this.prepareNocoData(updateObj, false, cookie);
const btForeignKeyColumn = columns.find(
(c) =>
c.uidt === UITypes.ForeignKey && data[c.column_name] !== undefined,
@ -4624,6 +4628,8 @@ class BaseModelSqlv2 {
NcError.recordNotFound(id);
}
await this.prepareNocoData(updateObj, false, cookie, prevData);
const query = this.dbDriver(this.tnPath)
.update(updateObj)
.where(await this._wherePk(id, true));
@ -5456,8 +5462,6 @@ class BaseModelSqlv2 {
continue;
}
if (!raw) {
await this.prepareNocoData(d, false, cookie);
pkAndData.push({
pk: pkValues,
data: d,
@ -5478,24 +5482,22 @@ class BaseModelSqlv2 {
},
);
if (oldRecords.length === tempToRead.length) {
prevData.push(...oldRecords);
} else {
for (const recordPk of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), recordPk),
);
for (const record of tempToRead) {
const oldRecord = oldRecords.find((r) =>
this.comparePks(this._extractPksValues(r), record.pk),
);
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(recordPk);
}
continue;
if (!oldRecord) {
// throw or skip if no record found
if (throwExceptionIfNotExist) {
NcError.recordNotFound(record);
}
prevData.push(oldRecord);
continue;
}
await this.prepareNocoData(record.data, false, cookie, oldRecord);
prevData.push(oldRecord);
}
for (const { pk, data } of tempToRead) {
@ -5809,6 +5811,11 @@ class BaseModelSqlv2 {
await transaction.commit();
await this.clearFileReferences({
oldData: deleted,
columns: this.model.columns,
});
if (isSingleRecordDeletion) {
await this.afterDelete(deleted[0], null, cookie);
} else {
@ -9136,7 +9143,12 @@ class BaseModelSqlv2 {
await this.execAndParse(qb, null, { raw: true });
}
async prepareNocoData(data, isInsertData = false, cookie?: { user?: any }) {
async prepareNocoData(
data,
isInsertData = false,
cookie?: { user?: any },
oldData?,
) {
for (const column of this.model.columns) {
if (
![
@ -9166,7 +9178,7 @@ class BaseModelSqlv2 {
}
}
if (column.uidt === UITypes.Attachment) {
if (data[column.column_name]) {
if (data && data[column.column_name]) {
try {
if (typeof data[column.column_name] === 'string') {
data[column.column_name] = JSON.parse(data[column.column_name]);
@ -9174,31 +9186,113 @@ class BaseModelSqlv2 {
} catch (e) {
NcError.invalidAttachmentJson(data[column.column_name]);
}
}
if (Array.isArray(data[column.column_name])) {
const sanitizedAttachments = [];
for (const attachment of data[column.column_name]) {
if (!('url' in attachment) && !('path' in attachment)) {
NcError.unprocessableEntity(
'Attachment object must contain either url or path',
);
}
sanitizedAttachments.push(
extractProps(attachment, [
'url',
'path',
'title',
'mimetype',
'size',
'icon',
'width',
'height',
]),
if (oldData && oldData[column.column_name]) {
try {
if (typeof oldData[column.column_name] === 'string') {
oldData[column.column_name] = JSON.parse(
oldData[column.column_name],
);
}
} catch (e) {}
}
const regenerateIds = [];
if (!isInsertData) {
const oldAttachmentMap = new Map<
string,
{ url?: string; path?: string }
>(
oldData &&
oldData[column.column_name] &&
Array.isArray(oldData[column.column_name])
? oldData[column.column_name].map((att) => [att.id, att])
: [],
);
const newAttachmentMap = new Map<
string,
{ url?: string; path?: string }
>(
data[column.column_name] && Array.isArray(data[column.column_name])
? data[column.column_name].map((att) => [att.id, att])
: [],
);
for (const [oldId, oldAttachment] of oldAttachmentMap) {
if (!newAttachmentMap.has(oldId)) {
await FileReference.delete(this.context, oldId);
} else if (
(oldAttachment.url &&
oldAttachment.url !== newAttachmentMap.get(oldId).url) ||
(oldAttachment.path &&
oldAttachment.path !== newAttachmentMap.get(oldId).path)
) {
await FileReference.delete(this.context, oldId);
regenerateIds.push(oldId);
}
}
for (const [newId, newAttachment] of newAttachmentMap) {
if (!oldAttachmentMap.has(newId)) {
regenerateIds.push(newId);
} else if (
(newAttachment.url &&
newAttachment.url !== oldAttachmentMap.get(newId).url) ||
(newAttachment.path &&
newAttachment.path !== oldAttachmentMap.get(newId).path)
) {
regenerateIds.push(newId);
}
}
}
const sanitizedAttachments = [];
if (Array.isArray(data[column.column_name])) {
for (const attachment of data[column.column_name]) {
if (!('url' in attachment) && !('path' in attachment)) {
NcError.unprocessableEntity(
'Attachment object must contain either url or path',
);
}
data[column.column_name] = JSON.stringify(sanitizedAttachments);
const sanitizedAttachment = extractProps(attachment, [
'id',
'url',
'path',
'title',
'mimetype',
'size',
'icon',
'width',
'height',
]);
if (
isInsertData ||
!sanitizedAttachment.id ||
regenerateIds.includes(sanitizedAttachment.id)
) {
sanitizedAttachment.id = await FileReference.insert(
this.context,
{
file_url: sanitizedAttachment.url ?? sanitizedAttachment.path,
file_size: sanitizedAttachment.size,
fk_user_id: cookie?.user?.id,
fk_model_id: this.model.id,
fk_column_id: column.id,
},
);
}
sanitizedAttachments.push(sanitizedAttachment);
}
}
data[column.column_name] = sanitizedAttachments.length
? JSON.stringify(sanitizedAttachments)
: null;
} else if (
[UITypes.User, UITypes.CreatedBy, UITypes.LastModifiedBy].includes(
column.uidt,
@ -9372,6 +9466,50 @@ class BaseModelSqlv2 {
qb,
);
}
protected async clearFileReferences(args: {
oldData?: Record<string, any>[];
columns?: Column[];
}) {
const { oldData, columns } = args;
const modelColumns = columns || (await this.model.getColumns(this.context));
const attachmentColumns = modelColumns.filter(
(c) => c.uidt === UITypes.Attachment,
);
if (attachmentColumns.length === 0) return;
for (const column of attachmentColumns) {
const oldAttachments = [];
if (oldData) {
for (const row of oldData) {
let attachmentRecord = row[column.title];
if (attachmentRecord) {
try {
if (typeof attachmentRecord === 'string') {
attachmentRecord = JSON.parse(row[column.title]);
}
for (const attachment of attachmentRecord) {
oldAttachments.push(attachment);
}
} catch (e) {
logger.error(e);
}
}
}
}
if (oldAttachments.length === 0) continue;
await FileReference.delete(
this.context,
oldAttachments.filter((at) => at.id).map((at) => at.id),
);
}
}
}
export function extractSortsObject(

59
packages/nocodb/src/meta/meta.service.ts
Unescape View File

@ -209,6 +209,64 @@ export class MetaService {
return insertObj;
}
/***
* Update multiple records in meta data
* @param workspace_id - Workspace id
* @param base_id - Base id
* @param target - Table name
* @param data - Data to be updated
* @param ids - Ids of the records to be updated
*/
public async bulkMetaUpdate(
workspace_id: string,
base_id: string,
target: string,
data: any | any[],
ids: string[],
): Promise<any> {
if (Array.isArray(data) ? !data.length : !data) {
return [];
}
const query = this.knexConnection(target);
const at = this.now();
if (workspace_id === base_id) {
if (!Object.values(RootScopes).includes(workspace_id as RootScopes)) {
NcError.metaError({
message: 'Invalid scope',
sql: '',
});
}
if (!RootScopeTables[workspace_id].includes(target)) {
NcError.metaError({
message: 'Table not accessible from this scope',
sql: '',
});
}
} else {
if (!base_id) {
NcError.metaError({
message: 'Base ID is required',
sql: '',
});
}
this.contextCondition(query, workspace_id, base_id, target);
}
const updateObj = {
...data,
updated_at: at,
};
query.whereIn('id', ids).update(updateObj);
return query;
}
/***
* Generate nanoid for the given target
* @param target - Table name
@ -252,6 +310,7 @@ export class MetaService {
[MetaTable.USER_COMMENTS_NOTIFICATIONS_PREFERENCE]: 'cnp',
[MetaTable.JOBS]: 'job',
[MetaTable.INTEGRATIONS]: 'int',
[MetaTable.FILE_REFERENCES]: 'at',
};
const prefix = prefixMap[target] || 'nc';

4
packages/nocodb/src/meta/migrations/XcMigrationSourcev2.ts
Unescape View File

@ -43,6 +43,7 @@ import * as nc_053_jobs from '~/meta/migrations/v2/nc_053_jobs';
import * as nc_054_id_length from '~/meta/migrations/v2/nc_054_id_length';
import * as nc_055_junction_pk from '~/meta/migrations/v2/nc_055_junction_pk';
import * as nc_056_integration from '~/meta/migrations/v2/nc_056_integration';
import * as nc_057_file_references from '~/meta/migrations/v2/nc_057_file_references';
// Create a custom migration source class
export default class XcMigrationSourcev2 {
@ -97,6 +98,7 @@ export default class XcMigrationSourcev2 {
'nc_054_id_length',
'nc_055_junction_pk',
'nc_056_integration',
'nc_057_file_references',
]);
}
@ -196,6 +198,8 @@ export default class XcMigrationSourcev2 {
return nc_055_junction_pk;
case 'nc_056_integration':
return nc_056_integration;
case 'nc_057_file_references':
return nc_057_file_references;
}
}
}

26
packages/nocodb/src/meta/migrations/v2/nc_057_file_references.ts
Unescape View File

@ -0,0 +1,26 @@
import type { Knex } from 'knex';
import { MetaTable } from '~/utils/globals';
const up = async (knex: Knex) => {
await knex.schema.createTable(MetaTable.FILE_REFERENCES, (table) => {
table.string('id', 20).primary().notNullable();
table.string('storage');
table.text('file_url');
table.integer('file_size');
table.string('fk_user_id', 20);
table.string('fk_workspace_id', 20);
table.string('base_id', 20);
table.string('fk_model_id', 20);
table.string('fk_column_id', 20);
table.boolean('deleted').defaultTo(false);
table.timestamps(true, true);
// TODO: Add indexes
});
};
const down = async (knex: Knex) => {
await knex.schema.dropTable(MetaTable.FILE_REFERENCES);
};
export { up, down };

147
packages/nocodb/src/models/FileReference.ts
Unescape View File

@ -0,0 +1,147 @@
import type { NcContext } from '~/interface/config';
import Noco from '~/Noco';
import { MetaTable } from '~/utils/globals';
import { extractProps } from '~/helpers/extractProps';
export default class FileReference {
id: string;
storage: string;
file_url: string;
file_size: number;
fk_user_id: string;
fk_workspace_id: string;
base_id: string;
fk_model_id: string;
fk_column_id: string;
deleted: boolean;
created_at: Date;
updated_at: Date;
constructor(data: Partial<FileReference>) {
Object.assign(this, data);
}
public static async insert(
context: NcContext,
fileRefObj: Partial<FileReference>,
ncMeta = Noco.ncMeta,
) {
const insertObj = extractProps(fileRefObj, [
'storage',
'file_url',
'file_size',
'fk_user_id',
'fk_model_id',
'fk_column_id',
'deleted',
]);
const { id } = await ncMeta.metaInsert2(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
insertObj,
);
return id;
}
public static async bulkInsert(
context: NcContext,
fileRefObjs: Partial<FileReference>[],
ncMeta = Noco.ncMeta,
) {
let insertObjs = fileRefObjs.map((fileRefObj) =>
extractProps(fileRefObj, [
'storage',
'file_url',
'file_size',
'fk_user_id',
'fk_model_id',
'fk_column_id',
'deleted',
]),
);
// insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES);
// use promise.all to populate the ids
insertObjs = await Promise.all(
insertObjs.map(async (insertObj) => {
insertObj.id = await ncMeta.genNanoid(MetaTable.FILE_REFERENCES);
return insertObj;
}),
);
await ncMeta.bulkMetaInsert(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
insertObjs,
);
return insertObjs;
}
public static async update(
context: NcContext,
fileReferenceId: string | string[],
fileReferenceObj: Partial<FileReference>,
ncMeta = Noco.ncMeta,
) {
const updateObj = extractProps(fileReferenceObj, ['deleted']);
fileReferenceId = Array.isArray(fileReferenceId)
? fileReferenceId
: [fileReferenceId];
return ncMeta.bulkMetaUpdate(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
updateObj,
fileReferenceId,
);
}
public static async delete(
context: NcContext,
fileReferenceId: string | string[],
ncMeta = Noco.ncMeta,
) {
fileReferenceId = Array.isArray(fileReferenceId)
? fileReferenceId
: [fileReferenceId];
await ncMeta.bulkMetaUpdate(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
{ deleted: true },
fileReferenceId,
);
}
public static async hardDelete(
context: NcContext,
fileReferenceId: string,
ncMeta = Noco.ncMeta,
) {
await ncMeta.metaDelete(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
fileReferenceId,
);
}
public static async get(context: NcContext, id: any, ncMeta = Noco.ncMeta) {
const fileReferenceData = await ncMeta.metaGet2(
context.workspace_id,
context.base_id,
MetaTable.FILE_REFERENCES,
id,
);
return fileReferenceData && new FileReference(fileReferenceData);
}
}

1
packages/nocodb/src/models/index.ts
Unescape View File

@ -45,3 +45,4 @@ export { default as Extension } from './Extension';
export { default as Comment } from './Comment';
export { default as Job } from './Job';
export { default as Integration } from './Integration';
export { default as FileReference } from './FileReference';

33
packages/nocodb/src/services/attachments.service.ts
Unescape View File

@ -14,7 +14,7 @@ import type Sharp from 'sharp';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
import mimetypes, { mimeIcons } from '~/utils/mimeTypes';
import { PresignedUrl } from '~/models';
import { FileReference, PresignedUrl } from '~/models';
import { utf8ify } from '~/helpers/stringHelpers';
import { NcError } from '~/helpers/catchError';
import { IJobsService } from '~/modules/jobs/jobs-service.interface';
@ -114,6 +114,21 @@ export class AttachmentsService {
file,
);
await FileReference.insert(
{
workspace_id: RootScopes.ROOT,
base_id: RootScopes.ROOT,
},
{
storage: storageAdapter.constructor.name,
file_url:
url ?? path.join('download', filePath.join('/'), fileName),
file_size: file.size,
fk_user_id: userId,
deleted: true, // root file references are always deleted as they are not associated with any record
},
);
const attachment: AttachmentObject = {
...(url
? { url }
@ -228,6 +243,22 @@ export class AttachmentsService {
},
);
await FileReference.insert(
{
workspace_id: RootScopes.ROOT,
base_id: RootScopes.ROOT,
},
{
storage: storageAdapter.constructor.name,
file_url:
attachmentUrl ??
path.join('download', filePath.join('/'), fileName),
file_size: file.size,
fk_user_id: userId,
deleted: true, // root file references are always deleted as they are not associated with any record
},
);
const tempMetadata: {
width?: number;
height?: number;

2
packages/nocodb/src/utils/globals.ts
Unescape View File

@ -53,6 +53,7 @@ export enum MetaTable {
COMMENTS_REACTIONS = 'nc_comment_reactions',
JOBS = 'nc_jobs',
INTEGRATIONS = 'nc_integrations_v2',
FILE_REFERENCES = 'nc_file_references',
}
export enum MetaTableOldV2 {
@ -286,6 +287,7 @@ export const RootScopeTables = {
MetaTable.STORE,
MetaTable.NOTIFICATION,
MetaTable.JOBS,
MetaTable.FILE_REFERENCES,
// Temporarily added need to be discussed within team
MetaTable.AUDIT,
],

Write Preview
Loading…
Cancel
Save