From cb3a9cfb0c589d873707f608ea3863bf09db8a0d Mon Sep 17 00:00:00 2001 From: Anbarasu Date: Fri, 26 Jul 2024 10:14:34 +0530 Subject: [PATCH] feat: thumbnail generator (#8974) * feat: thumbnail-processor * fix: use signedPath instead of path * fix: rebase * fix: minor fix * fix: url path * fix: use thumbnails for attachments carousel nav --- .../components/cell/attachment/Carousel.vue | 2 +- .../components/cell/attachment/Modal.vue | 4 +- .../cell/attachment/Preview/Image.vue | 15 +- .../components/cell/attachment/index.vue | 27 ++- .../nc-gui/components/smartsheet/Gallery.vue | 2 +- .../nc-gui/components/smartsheet/Kanban.vue | 2 +- .../smartsheet/calendar/SideMenu.vue | 2 +- .../virtual-cell/components/ListItem.vue | 2 +- packages/nc-gui/composables/useAttachment.ts | 6 +- packages/nocodb/.gitignore | 1 + .../attachments-secure.controller.ts | 9 +- .../attachments.controller.spec.ts | 21 -- .../src/controllers/attachments.controller.ts | 17 +- packages/nocodb/src/db/BaseModelSqlv2.ts | 197 +++++++++++++++++- packages/nocodb/src/interface/Jobs.ts | 7 +- .../jobs/fallback/fallback-queue.service.ts | 6 + .../nocodb/src/modules/jobs/jobs.module.ts | 2 + .../thumbnail-generator.processor.ts | 152 ++++++++++++++ packages/nocodb/src/modules/noco.module.ts | 5 +- .../nocodb/src/plugins/backblaze/Backblaze.ts | 60 +++--- packages/nocodb/src/plugins/gcs/Gcs.ts | 31 ++- .../src/plugins/linode/LinodeObjectStorage.ts | 60 +++--- packages/nocodb/src/plugins/mino/Minio.ts | 70 ++++--- .../nocodb/src/plugins/ovhCloud/OvhCloud.ts | 65 +++--- packages/nocodb/src/plugins/s3/S3.ts | 19 +- .../plugins/scaleway/ScalewayObjectStorage.ts | 60 +++--- packages/nocodb/src/plugins/spaces/Spaces.ts | 60 +++--- packages/nocodb/src/plugins/storage/Local.ts | 25 +-- .../nocodb/src/plugins/upcloud/UpoCloud.ts | 60 +++--- packages/nocodb/src/plugins/vultr/Vultr.ts | 60 +++--- .../src/services/attachments.service.ts | 77 ++++++- .../src/services/public-datas.service.ts | 64 +++++- 32 files changed, 874 insertions(+), 316 deletions(-) delete mode 100644 packages/nocodb/src/controllers/attachments.controller.spec.ts create mode 100644 packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts diff --git a/packages/nc-gui/components/cell/attachment/Carousel.vue b/packages/nc-gui/components/cell/attachment/Carousel.vue index 0fcfd7541d..b96e847e23 100644 --- a/packages/nc-gui/components/cell/attachment/Carousel.vue +++ b/packages/nc-gui/components/cell/attachment/Carousel.vue @@ -265,7 +265,7 @@ const initEmblaApi = (val: any) => { class="nc-attachment-img-wrapper h-12" object-fit="contain" :alt="item.title" - :srcs="getPossibleAttachmentSrc(item)" + :srcs="getPossibleAttachmentSrc(item, 'tiny')" />
{ modalVisible.value = false @@ -152,7 +152,7 @@ const handleFileDelete = (i: number) => { > index.value++ diff --git a/packages/nc-gui/components/smartsheet/calendar/SideMenu.vue b/packages/nc-gui/components/smartsheet/calendar/SideMenu.vue index 27e6a1c436..9093d4b50b 100644 --- a/packages/nc-gui/components/smartsheet/calendar/SideMenu.vue +++ b/packages/nc-gui/components/smartsheet/calendar/SideMenu.vue @@ -586,7 +586,7 @@ onClickOutside(searchRef, toggleSearch) v-if="isImage(attachment.title, attachment.mimetype ?? attachment.type)" :key="`carousel-${record.row.id}-${index}`" class="h-10 !w-10 !object-contain" - :srcs="getPossibleAttachmentSrc(attachment)" + :srcs="getPossibleAttachmentSrc(attachment, 'tiny')" /> diff --git a/packages/nc-gui/components/virtual-cell/components/ListItem.vue b/packages/nc-gui/components/virtual-cell/components/ListItem.vue index 952f34a6f4..2553a86ced 100644 --- a/packages/nc-gui/components/virtual-cell/components/ListItem.vue +++ b/packages/nc-gui/components/virtual-cell/components/ListItem.vue @@ -92,7 +92,7 @@ const displayValue = computed(() => { v-if="isImage(attachmentObj.title, attachmentObj.mimetype ?? attachmentObj.type)" :key="`carousel-${attachmentObj.title}-${index}`" class="!w-11 !h-11 !max-h-11 !max-w-11object-cover !rounded-l-xl" - :srcs="getPossibleAttachmentSrc(attachmentObj)" + :srcs="getPossibleAttachmentSrc(attachmentObj, 'tiny')" /> diff --git a/packages/nc-gui/composables/useAttachment.ts b/packages/nc-gui/composables/useAttachment.ts index dc81ad0ffa..3dad0704a4 100644 --- a/packages/nc-gui/composables/useAttachment.ts +++ b/packages/nc-gui/composables/useAttachment.ts @@ -1,8 +1,12 @@ const useAttachment = () => { const { appInfo } = useGlobal() - const getPossibleAttachmentSrc = (item: Record) => { + const getPossibleAttachmentSrc = (item: Record, thumbnail?: 'card_cover' | 'tiny' | 'small') => { const res: string[] = [] + + if (thumbnail && item?.thumbnails && item.thumbnails[thumbnail]) { + res.push(getPossibleAttachmentSrc(item.thumbnails[thumbnail])[0]) + } if (item?.data) res.push(item.data) if (item?.file) res.push(window.URL.createObjectURL(item.file)) if (item?.signedPath) res.push(`${appInfo.value.ncSiteUrl}/${encodeURI(item.signedPath)}`) diff --git a/packages/nocodb/.gitignore b/packages/nocodb/.gitignore index ccece4a5b0..6780fa3ba5 100644 --- a/packages/nocodb/.gitignore +++ b/packages/nocodb/.gitignore @@ -1,6 +1,7 @@ # compiled output /dist /node_modules +nc # Logs logs diff --git a/packages/nocodb/src/controllers/attachments-secure.controller.ts b/packages/nocodb/src/controllers/attachments-secure.controller.ts index 46b2f2d4ca..ea29bf0fd0 100644 --- a/packages/nocodb/src/controllers/attachments-secure.controller.ts +++ b/packages/nocodb/src/controllers/attachments-secure.controller.ts @@ -1,4 +1,5 @@ import path from 'path'; +import fs from 'fs'; import { Body, Controller, @@ -96,10 +97,16 @@ export class AttachmentsSecureController { ); } + const filePath = param.split('/')[2] === 'thumbnails' ? '' : 'uploads'; + const file = await this.attachmentsService.getFile({ - path: path.join('nc', 'uploads', fpath), + path: path.join('nc', filePath, fpath), }); + if (!fs.existsSync(file.path)) { + return res.status(404).send('File not found'); + } + if (queryResponseContentType) { res.setHeader('Content-Type', queryResponseContentType); } diff --git a/packages/nocodb/src/controllers/attachments.controller.spec.ts b/packages/nocodb/src/controllers/attachments.controller.spec.ts deleted file mode 100644 index e4eb6d8edb..0000000000 --- a/packages/nocodb/src/controllers/attachments.controller.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { AttachmentsService } from '../services/attachments.service'; -import { AttachmentsController } from './attachments.controller'; -import type { TestingModule } from '@nestjs/testing'; - -describe('AttachmentsController', () => { - let controller: AttachmentsController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [AttachmentsController], - providers: [AttachmentsService], - }).compile(); - - controller = module.get(AttachmentsController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/packages/nocodb/src/controllers/attachments.controller.ts b/packages/nocodb/src/controllers/attachments.controller.ts index 3e6f8ec2c4..e019af4fd8 100644 --- a/packages/nocodb/src/controllers/attachments.controller.ts +++ b/packages/nocodb/src/controllers/attachments.controller.ts @@ -1,4 +1,5 @@ import path from 'path'; +import fs from 'fs'; import { Body, Controller, @@ -83,6 +84,10 @@ export class AttachmentsController { path: path.join('nc', 'uploads', filename), }); + if (!fs.existsSync(file.path)) { + return res.status(404).send('File not found'); + } + if (isPreviewAllowed({ mimetype: file.type, path: file.path })) { if (queryFilename) { res.setHeader( @@ -120,6 +125,10 @@ export class AttachmentsController { ), }); + if (!fs.existsSync(file.path)) { + return res.status(404).send('File not found'); + } + if (isPreviewAllowed({ mimetype: file.type, path: file.path })) { if (queryFilename) { res.setHeader( @@ -156,10 +165,16 @@ export class AttachmentsController { ); } + const filePath = param.split('/')[2] === 'thumbnails' ? '' : 'uploads'; + const file = await this.attachmentsService.getFile({ - path: path.join('nc', 'uploads', fpath), + path: path.join('nc', filePath, fpath), }); + if (!fs.existsSync(file.path)) { + return res.status(404).send('File not found'); + } + if (queryResponseContentType) { res.setHeader('Content-Type', queryResponseContentType); } diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index 15c342385a..b6db122e5a 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -7916,47 +7916,228 @@ class BaseModelSqlv2 { if (Array.isArray(attachment)) { for (const lookedUpAttachment of attachment) { if (lookedUpAttachment?.path) { + let relativePath = lookedUpAttachment.path.replace( + /^download\//, + '', + ); + promises.push( PresignedUrl.getSignedUrl({ - pathOrUrl: lookedUpAttachment.path.replace( - /^download\//, - '', - ), + pathOrUrl: relativePath, preview: true, mimetype: lookedUpAttachment.mimetype, filename: lookedUpAttachment.title, }).then((r) => (lookedUpAttachment.signedPath = r)), ); + + if (!lookedUpAttachment.mimetype?.startsWith('image/')) { + continue; + } + + lookedUpAttachment.thumbnails = { + tiny: {}, + small: {}, + card_cover: {}, + }; + + relativePath = `thumbnails/${relativePath}`; + + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/tiny.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.tiny.signedPath = r), + ), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/small.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.small.signedPath = r), + ), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/card_cover.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.card_cover.signedPath = + r), + ), + ); } else if (lookedUpAttachment?.url) { + let relativePath = lookedUpAttachment.url; promises.push( PresignedUrl.getSignedUrl({ - pathOrUrl: lookedUpAttachment.url, + pathOrUrl: relativePath, preview: true, mimetype: lookedUpAttachment.mimetype, filename: lookedUpAttachment.title, }).then((r) => (lookedUpAttachment.signedUrl = r)), ); + + if (!lookedUpAttachment.mimetype?.startsWith('image/')) { + continue; + } + + relativePath = relativePath.replace( + 'nc/uploads', + 'nc/thumbnails', + ); + + lookedUpAttachment.thumbnails = { + tiny: {}, + small: {}, + card_cover: {}, + }; + + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/tiny.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.tiny.signedUrl = r), + ), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/small.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.small.signedUrl = r), + ), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/card_cover.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: lookedUpAttachment.title, + }).then( + (r) => + (lookedUpAttachment.thumbnails.card_cover.signedUrl = + r), + ), + ); } } } else { if (attachment?.path) { + let relativePath = attachment.path.replace(/^download\//, ''); + promises.push( PresignedUrl.getSignedUrl({ - pathOrUrl: attachment.path.replace(/^download\//, ''), + pathOrUrl: relativePath, preview: true, mimetype: attachment.mimetype, filename: attachment.title, }).then((r) => (attachment.signedPath = r)), ); + if (!attachment.mimetype?.startsWith('image/')) { + continue; + } + + relativePath = `thumbnails/${relativePath}`; + + attachment.thumbnails = { + tiny: {}, + small: {}, + card_cover: {}, + }; + + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/tiny.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then((r) => (attachment.thumbnails.tiny.signedPath = r)), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/small.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then( + (r) => (attachment.thumbnails.small.signedPath = r), + ), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/card_cover.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then( + (r) => (attachment.thumbnails.card_cover.signedPath = r), + ), + ); } else if (attachment?.url) { + let relativePath = attachment.url; + promises.push( PresignedUrl.getSignedUrl({ - pathOrUrl: attachment.url, + pathOrUrl: relativePath, preview: true, mimetype: attachment.mimetype, filename: attachment.title, }).then((r) => (attachment.signedUrl = r)), ); + + relativePath = relativePath.replace( + 'nc/uploads', + 'nc/thumbnails', + ); + attachment.thumbnails = { + tiny: {}, + small: {}, + card_cover: {}, + }; + + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/tiny.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then((r) => (attachment.thumbnails.tiny.signedUrl = r)), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/small.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then((r) => (attachment.thumbnails.small.signedUrl = r)), + ); + promises.push( + PresignedUrl.getSignedUrl({ + pathOrUrl: `${relativePath}/card_cover.jpg`, + preview: true, + mimetype: 'image/jpeg', + filename: attachment.title, + }).then( + (r) => (attachment.thumbnails.card_cover.signedUrl = r), + ), + ); } } } @@ -9097,6 +9278,8 @@ class BaseModelSqlv2 { 'mimetype', 'size', 'icon', + 'width', + 'height', ]), ); } diff --git a/packages/nocodb/src/interface/Jobs.ts b/packages/nocodb/src/interface/Jobs.ts index c3d3ad765e..307f0038d8 100644 --- a/packages/nocodb/src/interface/Jobs.ts +++ b/packages/nocodb/src/interface/Jobs.ts @@ -1,4 +1,4 @@ -import type { UserType } from 'nocodb-sdk'; +import type { AttachmentResType, UserType } from 'nocodb-sdk'; import type { NcContext, NcRequest } from '~/interface/config'; export const JOBS_QUEUE = 'jobs'; @@ -17,6 +17,7 @@ export enum JobTypes { HandleWebhook = 'handle-webhook', CleanUp = 'clean-up', DataExport = 'data-export', + ThumbnailGenerator = 'thumbnail-generator', } export enum JobStatus { @@ -121,3 +122,7 @@ export interface DataExportJobData extends JobData { exportAs: 'csv' | 'json' | 'xlsx'; ncSiteUrl: string; } + +export interface ThumbnailGeneratorJobData extends JobData { + attachments: AttachmentResType[]; +} diff --git a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts index 37b8323477..4bbf6b95bc 100644 --- a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts @@ -10,6 +10,7 @@ import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/web import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor'; import { JobsEventService } from '~/modules/jobs/jobs-event.service'; import { JobStatus, JobTypes } from '~/interface/Jobs'; +import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; export interface Job { id: string; @@ -35,6 +36,7 @@ export class QueueService { protected readonly sourceDeleteProcessor: SourceDeleteProcessor, protected readonly webhookHandlerProcessor: WebhookHandlerProcessor, protected readonly dataExportProcessor: DataExportProcessor, + protected readonly thumbnailGeneratorProcessor: ThumbnailGeneratorProcessor, ) { this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => { const job = this.queueMemory.find((job) => job.id === data.job.id); @@ -100,6 +102,10 @@ export class QueueService { this: this.dataExportProcessor, fn: this.dataExportProcessor.job, }, + [JobTypes.ThumbnailGenerator]: { + this: this.thumbnailGeneratorProcessor, + fn: this.thumbnailGeneratorProcessor.job, + }, }; async jobWrapper(job: Job) { diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index 6f718d7efe..88d962914f 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -18,6 +18,7 @@ import { SourceDeleteProcessor } from '~/modules/jobs/jobs/source-delete/source- import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/webhook-handler.processor'; import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor'; import { DataExportController } from '~/modules/jobs/jobs/data-export/data-export.controller'; +import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; // Jobs Module Related import { JobsLogService } from '~/modules/jobs/jobs/jobs-log.service'; @@ -78,6 +79,7 @@ export const JobsModuleMetadata = { SourceDeleteProcessor, WebhookHandlerProcessor, DataExportProcessor, + ThumbnailGeneratorProcessor, ], exports: ['JobsService'], }; diff --git a/packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts b/packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts new file mode 100644 index 0000000000..8513c7454a --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts @@ -0,0 +1,152 @@ +import path from 'path'; +import { Readable } from 'stream'; +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { Logger } from '@nestjs/common'; +import sharp from 'sharp'; +import axios from 'axios'; +import type { AttachmentResType } from 'nocodb-sdk'; +import type { ThumbnailGeneratorJobData } from '~/interface/Jobs'; +import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs'; +import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; +import { PresignedUrl } from '~/models'; +import { AttachmentsService } from '~/services/attachments.service'; + +const attachmentPreviews = ['image/']; + +@Processor(JOBS_QUEUE) +export class ThumbnailGeneratorProcessor { + constructor(private readonly attachmentsService: AttachmentsService) {} + + private logger = new Logger(ThumbnailGeneratorProcessor.name); + + @Process(JobTypes.ThumbnailGenerator) + async job(job: Job) { + try { + const { attachments } = job.data; + + const thumbnailPromises = attachments + .filter((attachment) => + attachmentPreviews.some((type) => + attachment.mimetype.startsWith(type), + ), + ) + .map(async (attachment) => { + const thumbnail = await this.generateThumbnail(attachment); + return { + path: attachment.path ?? attachment.url, + card_cover: thumbnail?.card_cover, + small: thumbnail?.small, + tiny: thumbnail?.tiny, + }; + }); + + return await Promise.all(thumbnailPromises); + } catch (error) { + this.logger.error('Failed to generate thumbnails', error); + } + } + + private async generateThumbnail( + attachment: AttachmentResType, + ): Promise<{ [key: string]: string }> { + const { file, relativePath } = await this.getFileData(attachment); + + const thumbnailPaths = { + card_cover: path.join('nc', 'thumbnails', relativePath, 'card_cover.jpg'), + small: path.join('nc', 'thumbnails', relativePath, 'small.jpg'), + tiny: path.join('nc', 'thumbnails', relativePath, 'tiny.jpg'), + }; + + try { + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + + await Promise.all( + Object.entries(thumbnailPaths).map(async ([size, thumbnailPath]) => { + let height; + switch (size) { + case 'card_cover': + height = 512; + break; + case 'small': + height = 128; + break; + case 'tiny': + height = 64; + break; + default: + height = 32; + break; + } + + const resizedImage = await sharp(file, { + limitInputPixels: false, + }) + .resize(undefined, height, { + fit: sharp.fit.cover, + kernel: 'lanczos3', + }) + .toBuffer(); + + await (storageAdapter as any).fileCreateByStream( + thumbnailPath, + Readable.from(resizedImage), + { + mimetype: 'image/jpeg', + }, + ); + }), + ); + + return thumbnailPaths; + } catch (error) { + this.logger.error( + `Failed to generate thumbnails for ${attachment.path}`, + error, + ); + } + } + + private async getFileData( + attachment: AttachmentResType, + ): Promise<{ file: Buffer; relativePath: string }> { + let url, signedUrl, file; + let relativePath; + + if (attachment.path) { + relativePath = attachment.path.replace(/^download\//, ''); + url = await PresignedUrl.getSignedUrl({ + pathOrUrl: relativePath, + preview: false, + filename: attachment.title, + mimetype: attachment.mimetype, + }); + + const fullPath = await PresignedUrl.getPath(`${url}`); + const [fpath] = fullPath.split('?'); + const tempPath = await this.attachmentsService.getFile({ + path: path.join('nc', 'uploads', fpath), + }); + relativePath = fpath; + file = tempPath.path; + } else if (attachment.url) { + relativePath = decodeURI(new URL(attachment.url).pathname); + + signedUrl = await PresignedUrl.getSignedUrl({ + pathOrUrl: relativePath, + preview: false, + filename: attachment.title, + mimetype: attachment.mimetype, + }); + + file = (await axios({ url: signedUrl, responseType: 'arraybuffer' })) + .data as Buffer; + } + + if (relativePath.startsWith('/nc/uploads/')) { + relativePath = relativePath.replace('/nc/uploads/', ''); + } + + return { file, relativePath }; + } +} diff --git a/packages/nocodb/src/modules/noco.module.ts b/packages/nocodb/src/modules/noco.module.ts index 7866bd16b5..ba616f6c45 100644 --- a/packages/nocodb/src/modules/noco.module.ts +++ b/packages/nocodb/src/modules/noco.module.ts @@ -1,8 +1,8 @@ -import multer from 'multer'; import { Module } from '@nestjs/common'; -import { MulterModule } from '@nestjs/platform-express'; /* Modules */ +import { MulterModule } from '@nestjs/platform-express'; +import multer from 'multer'; import { EventEmitterModule } from '~/modules/event-emitter/event-emitter.module'; import { JobsModule } from '~/modules/jobs/jobs.module'; @@ -181,7 +181,6 @@ export const nocoModuleMetadata = { CommandPaletteController, ExtensionsController, JobsMetaController, - /* Datas */ DataTableController, DatasController, diff --git a/packages/nocodb/src/plugins/backblaze/Backblaze.ts b/packages/nocodb/src/plugins/backblaze/Backblaze.ts index 351ace6964..eca5d79b76 100644 --- a/packages/nocodb/src/plugins/backblaze/Backblaze.ts +++ b/packages/nocodb/src/plugins/backblaze/Backblaze.ts @@ -16,31 +16,10 @@ export default class Backblaze implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -69,7 +48,10 @@ export default class Backblaze implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -79,9 +61,31 @@ export default class Backblaze implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Body: stream, + Key: key, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/gcs/Gcs.ts b/packages/nocodb/src/plugins/gcs/Gcs.ts index c3033e5454..4eb9032570 100644 --- a/packages/nocodb/src/plugins/gcs/Gcs.ts +++ b/packages/nocodb/src/plugins/gcs/Gcs.ts @@ -22,6 +22,7 @@ export default class Gcs implements IStorageAdapterV2 { .bucket(this.bucketName) .upload(file.path, { destination: key, + contentType: file?.mimetype || 'application/octet-stream', // Support for HTTP requests made with `Accept-Encoding: gzip` gzip: true, // By setting the option `destination`, you can change the name of the @@ -118,7 +119,7 @@ export default class Gcs implements IStorageAdapterV2 { .bucket(this.bucketName) .file(destPath) .save(response.data) - .then((res) => resolve(res)) + .then((res) => resolve({ url: res, data: response.data })) .catch(reject); }) .catch((error) => { @@ -127,9 +128,31 @@ export default class Gcs implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadResponse = await this.storageClient + .bucket(this.bucketName) + .file(key) + .save(stream, { + // Support for HTTP requests made with `Accept-Encoding: gzip` + gzip: true, + // By setting the option `destination`, you can change the name of the + // object you are uploading to a bucket. + metadata: { + contentType: options.mimetype || 'application/octet-stream', + // Enable long-lived HTTP caching headers + // Use only if the contents of the file will never change + // (If the contents will change, use cacheControl: 'no-cache') + cacheControl: 'public, max-age=31536000', + }, + }); + + return uploadResponse[0].publicUrl(); } // TODO - implement diff --git a/packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts b/packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts index b830a6c261..fee49f0b47 100644 --- a/packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts +++ b/packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts @@ -16,31 +16,10 @@ export default class LinodeObjectStorage implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -68,7 +47,10 @@ export default class LinodeObjectStorage implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -78,9 +60,31 @@ export default class LinodeObjectStorage implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + public async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Body: stream, + Key: key, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/mino/Minio.ts b/packages/nocodb/src/plugins/mino/Minio.ts index ea2bdd58f0..fe60ff8cba 100644 --- a/packages/nocodb/src/plugins/mino/Minio.ts +++ b/packages/nocodb/src/plugins/mino/Minio.ts @@ -16,32 +16,9 @@ export default class Minio implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); - - // uploadParams.Body = fileStream; - // uploadParams.Key = key; - const metaData = { - 'Content-Type': file.mimetype, - // 'X-Amz-Meta-Testing': 1234, - // 'run': 5678 - }; - // call S3 to retrieve upload file to specified bucket - this.minioClient - .putObject(this.input?.bucket, key, fileStream, metaData) - .then(() => { - resolve( - `http${this.input.useSSL ? 's' : ''}://${this.input.endPoint}:${ - this.input.port - }/${this.input.bucket}/${key}`, - ); - }) - .catch(reject); + const fileStream = fs.createReadStream(file.path); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -117,11 +94,12 @@ export default class Minio implements IStorageAdapterV2 { this.minioClient .putObject(this.input?.bucket, key, response.data, metaData) .then(() => { - resolve( - `http${this.input.useSSL ? 's' : ''}://${this.input.endPoint}:${ - this.input.port - }/${this.input.bucket}/${key}`, - ); + resolve({ + url: `http${this.input.useSSL ? 's' : ''}://${ + this.input.endPoint + }:${this.input.port}/${this.input.bucket}/${key}`, + data: response.data, + }); }) .catch(reject); }) @@ -131,9 +109,33 @@ export default class Minio implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + return new Promise((resolve, reject) => { + // uploadParams.Body = fileStream; + // uploadParams.Key = key; + const metaData = { + 'Content-Type': options?.mimetype, + // 'X-Amz-Meta-Testing': 1234, + // 'run': 5678 + }; + // call S3 to retrieve upload file to specified bucket + this.minioClient + .putObject(this.input?.bucket, key, stream, metaData) + .then(() => { + resolve( + `http${this.input.useSSL ? 's' : ''}://${this.input.endPoint}:${ + this.input.port + }/${this.input.bucket}/${key}`, + ); + }) + .catch(reject); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts b/packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts index e1114f4a34..172d98bc2b 100644 --- a/packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts +++ b/packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts @@ -16,31 +16,10 @@ export default class OvhCloud implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); - - uploadParams.Body = fileStream; - uploadParams.Key = key; + const fileStream = fs.createReadStream(file.path); - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -68,7 +47,10 @@ export default class OvhCloud implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -78,9 +60,36 @@ export default class OvhCloud implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + // ContentType: file.mimetype, + }; + return new Promise((resolve, reject) => { + // Configure the file stream and obtain the upload parameters + + uploadParams.Body = stream; + uploadParams.Key = key; + uploadParams.ContentType = + options?.mimetype || 'application/octet-stream'; + + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/s3/S3.ts b/packages/nocodb/src/plugins/s3/S3.ts index 3632ffbeaf..3520ef1900 100644 --- a/packages/nocodb/src/plugins/s3/S3.ts +++ b/packages/nocodb/src/plugins/s3/S3.ts @@ -29,7 +29,9 @@ export default class S3 implements IStorageAdapterV2 { // create file stream const fileStream = fs.createReadStream(file.path); // upload using stream - return this.fileCreateByStream(key, fileStream); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, + }); } async fileCreateByUrl(key: string, url: string): Promise { @@ -49,13 +51,22 @@ export default class S3 implements IStorageAdapterV2 { uploadParams.ContentType = response.headers['content-type']; const data = await this.upload(uploadParams); - return data; + return { + url: data, + data: response.data, + }; } catch (error) { throw error; } } - fileCreateByStream(key: string, stream: Readable): Promise { + fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { const uploadParams: any = { ...this.defaultParams, }; @@ -67,6 +78,8 @@ export default class S3 implements IStorageAdapterV2 { uploadParams.Body = stream; uploadParams.Key = key; + uploadParams.ContentType = + options?.mimetype || 'application/octet-stream'; // call S3 to upload file to specified bucket this.upload(uploadParams) diff --git a/packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts b/packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts index 8c6459fb07..19999bd1b8 100644 --- a/packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts +++ b/packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts @@ -66,31 +66,10 @@ export default class ScalewayObjectStorage implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -118,7 +97,10 @@ export default class ScalewayObjectStorage implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -128,9 +110,31 @@ export default class ScalewayObjectStorage implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Body: stream, + Key: key, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/spaces/Spaces.ts b/packages/nocodb/src/plugins/spaces/Spaces.ts index e2f94c3835..f45d721a37 100644 --- a/packages/nocodb/src/plugins/spaces/Spaces.ts +++ b/packages/nocodb/src/plugins/spaces/Spaces.ts @@ -16,31 +16,10 @@ export default class Spaces implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -68,7 +47,10 @@ export default class Spaces implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -78,9 +60,31 @@ export default class Spaces implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Body: stream, + Key: key, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/storage/Local.ts b/packages/nocodb/src/plugins/storage/Local.ts index 9564122842..a4e163381f 100644 --- a/packages/nocodb/src/plugins/storage/Local.ts +++ b/packages/nocodb/src/plugins/storage/Local.ts @@ -30,7 +30,7 @@ export default class Local implements IStorageAdapterV2 { return new Promise((resolve, reject) => { axios .get(url, { - responseType: 'stream', + responseType: 'arraybuffer', headers: { accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', @@ -46,23 +46,16 @@ export default class Local implements IStorageAdapterV2 { }) .then(async (response) => { await mkdirp(path.dirname(destPath)); - const file = fs.createWriteStream(destPath); - // close() is async, call cb after close completes - file.on('finish', () => { - file.close((err) => { - if (err) { - return reject(err); - } - resolve(null); - }); - }); - file.on('error', (err) => { - // Handle errors - fs.unlink(destPath, () => reject(err.message)); // delete the (partial) file and then return the error + fs.writeFile(destPath, response.data, (err) => { + if (err) { + return reject(err); + } + resolve({ + url: null, + data: response.data, + }); }); - - response.data.pipe(file); }) .catch((err) => { reject(err.message); diff --git a/packages/nocodb/src/plugins/upcloud/UpoCloud.ts b/packages/nocodb/src/plugins/upcloud/UpoCloud.ts index ff6a31b4fb..780365d923 100644 --- a/packages/nocodb/src/plugins/upcloud/UpoCloud.ts +++ b/packages/nocodb/src/plugins/upcloud/UpoCloud.ts @@ -16,31 +16,10 @@ export default class UpoCloud implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -68,7 +47,10 @@ export default class UpoCloud implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -78,9 +60,31 @@ export default class UpoCloud implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype?: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Key: key, + Body: stream, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/plugins/vultr/Vultr.ts b/packages/nocodb/src/plugins/vultr/Vultr.ts index 0d3e79c946..d50fecc9e4 100644 --- a/packages/nocodb/src/plugins/vultr/Vultr.ts +++ b/packages/nocodb/src/plugins/vultr/Vultr.ts @@ -16,31 +16,10 @@ export default class Vultr implements IStorageAdapterV2 { } async fileCreate(key: string, file: XcFile): Promise { - const uploadParams: any = { - ACL: 'public-read', - ContentType: file.mimetype, - }; - return new Promise((resolve, reject) => { - // Configure the file stream and obtain the upload parameters - const fileStream = fs.createReadStream(file.path); - fileStream.on('error', (err) => { - console.log('File Error', err); - reject(err); - }); + const fileStream = fs.createReadStream(file.path); - uploadParams.Body = fileStream; - uploadParams.Key = key; - - // call S3 to retrieve upload file to specified bucket - this.s3Client.upload(uploadParams, (err, data) => { - if (err) { - console.log('Error', err); - reject(err); - } - if (data) { - resolve(data.Location); - } - }); + return this.fileCreateByStream(key, fileStream, { + mimetype: file?.mimetype, }); } @@ -68,7 +47,10 @@ export default class Vultr implements IStorageAdapterV2 { reject(err1); } if (data) { - resolve(data.Location); + resolve({ + url: data.Location, + data: response.data, + }); } }); }) @@ -78,9 +60,31 @@ export default class Vultr implements IStorageAdapterV2 { }); } - // TODO - implement - fileCreateByStream(_key: string, _stream: Readable): Promise { - return Promise.resolve(undefined); + async fileCreateByStream( + key: string, + stream: Readable, + options?: { + mimetype: string; + }, + ): Promise { + const uploadParams: any = { + ACL: 'public-read', + Body: stream, + Key: key, + ContentType: options?.mimetype || 'application/octet-stream', + }; + return new Promise((resolve, reject) => { + // call S3 to retrieve upload file to specified bucket + this.s3Client.upload(uploadParams, (err, data) => { + if (err) { + console.log('Error', err); + reject(err); + } + if (data) { + resolve(data.Location); + } + }); + }); } // TODO - implement diff --git a/packages/nocodb/src/services/attachments.service.ts b/packages/nocodb/src/services/attachments.service.ts index 11c243a5d4..430e1e7882 100644 --- a/packages/nocodb/src/services/attachments.service.ts +++ b/packages/nocodb/src/services/attachments.service.ts @@ -1,11 +1,12 @@ import path from 'path'; import Url from 'url'; import { AppEvents } from 'nocodb-sdk'; -import { Injectable, Logger } from '@nestjs/common'; +import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; import { nanoid } from 'nanoid'; import slash from 'slash'; import PQueue from 'p-queue'; import axios from 'axios'; +import sharp from 'sharp'; import type { AttachmentReqType, FileType } from 'nocodb-sdk'; import type { NcRequest } from '~/interface/config'; import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; @@ -15,6 +16,9 @@ import mimetypes, { mimeIcons } from '~/utils/mimeTypes'; import { PresignedUrl } from '~/models'; import { utf8ify } from '~/helpers/stringHelpers'; import { NcError } from '~/helpers/catchError'; +import { IJobsService } from '~/modules/jobs/jobs-service.interface'; +import { JobTypes } from '~/interface/Jobs'; +import { RootScopes } from '~/utils/globals'; interface AttachmentObject { url?: string; @@ -31,7 +35,11 @@ interface AttachmentObject { export class AttachmentsService { protected logger = new Logger(AttachmentsService.name); - constructor(private readonly appHooksService: AppHooksService) {} + constructor( + private readonly appHooksService: AppHooksService, + @Inject(forwardRef(() => 'JobsService')) + private readonly jobsService: IJobsService, + ) {} async upload(param: { path?: string; files: FileType[]; req: NcRequest }) { // TODO: add getAjvValidatorMw @@ -60,6 +68,26 @@ export class AttachmentsService { 5, )}${path.extname(originalName)}`; + const tempMetadata: { + width?: number; + height?: number; + } = {}; + + if (file.mimetype.includes('image')) { + try { + const metadata = await sharp(file.path, { + limitInputPixels: false, + }).metadata(); + + if (metadata.width && metadata.height) { + tempMetadata.width = metadata.width; + tempMetadata.height = metadata.height; + } + } catch (e) { + // Might be invalid image - ignore + } + } + const url = await storageAdapter.fileCreate( slash(path.join(destPath, fileName)), file, @@ -75,6 +103,7 @@ export class AttachmentsService { mimetype: file.mimetype, size: file.size, icon: mimeIcons[path.extname(originalName).slice(1)] || undefined, + ...tempMetadata, }; await this.signAttachment({ attachment }); @@ -95,6 +124,14 @@ export class AttachmentsService { throw errors[0]; } + await this.jobsService.add(JobTypes.ThumbnailGenerator, { + context: { + base_id: RootScopes.ROOT, + workspace_id: RootScopes.ROOT, + }, + attachments, + }); + this.appHooksService.emit(AppEvents.ATTACHMENT_UPLOAD, { type: 'file', req: param.req, @@ -140,10 +177,29 @@ export class AttachmentsService { 5, )}${path.extname(fileNameWithExt)}`; - const attachmentUrl = await storageAdapter.fileCreateByUrl( - slash(path.join(destPath, fileName)), - finalUrl, - ); + const { url: attachmentUrl, data: file } = + await storageAdapter.fileCreateByUrl( + slash(path.join(destPath, fileName)), + finalUrl, + ); + + const tempMetadata: { + width?: number; + height?: number; + } = {}; + + try { + const metadata = await sharp(file, { + limitInputPixels: true, + }).metadata(); + + if (metadata.width && metadata.height) { + tempMetadata.width = metadata.width; + tempMetadata.height = metadata.height; + } + } catch (e) { + // Might be invalid image - ignore + } let mimeType = response.headers['content-type']?.split(';')[0]; const size = response.headers['content-length']; @@ -163,6 +219,7 @@ export class AttachmentsService { size: size ? parseInt(size) : urlMeta.size, icon: mimeIcons[path.extname(fileNameWithExt).slice(1)] || undefined, + ...tempMetadata, }; await this.signAttachment({ attachment }); @@ -181,6 +238,14 @@ export class AttachmentsService { throw errors[0]; } + await this.jobsService.add(JobTypes.ThumbnailGenerator, { + context: { + base_id: RootScopes.ROOT, + workspace_id: RootScopes.ROOT, + }, + attachments, + }); + this.appHooksService.emit(AppEvents.ATTACHMENT_UPLOAD, { type: 'url', req: param.req, diff --git a/packages/nocodb/src/services/public-datas.service.ts b/packages/nocodb/src/services/public-datas.service.ts index 08ed430611..f2306097fc 100644 --- a/packages/nocodb/src/services/public-datas.service.ts +++ b/packages/nocodb/src/services/public-datas.service.ts @@ -1,10 +1,11 @@ import path from 'path'; -import { Injectable } from '@nestjs/common'; +import { forwardRef, Inject, Injectable } from '@nestjs/common'; import { nanoid } from 'nanoid'; import { populateUniqueFileName, UITypes, ViewTypes } from 'nocodb-sdk'; import slash from 'slash'; import { nocoExecute } from 'nc-help'; +import sharp from 'sharp'; import type { LinkToAnotherRecordColumn } from '~/models'; import type { NcContext } from '~/interface/config'; import { Column, Model, Source, View } from '~/models'; @@ -18,6 +19,9 @@ import { mimeIcons } from '~/utils/mimeTypes'; import { utf8ify } from '~/helpers/stringHelpers'; import { replaceDynamicFieldWithValue } from '~/db/BaseModelSqlv2'; import { Filter } from '~/models'; +import { IJobsService } from '~/modules/jobs/jobs-service.interface'; +import { JobTypes } from '~/interface/Jobs'; +import { RootScopes } from '~/utils/globals'; import { DatasService } from '~/services/datas.service'; // todo: move to utils @@ -27,7 +31,12 @@ export function sanitizeUrlPath(paths) { @Injectable() export class PublicDatasService { - constructor(protected datasService: DatasService) {} + constructor( + protected datasService: DatasService, + @Inject(forwardRef(() => 'JobsService')) + protected readonly jobsService: IJobsService, + ) {} + async dataList( context: NcContext, param: { @@ -407,6 +416,23 @@ export class PublicDatasService { attachments[fieldName].map((att) => att?.title), ); + const tempMeta: { + width?: number; + height?: number; + } = {}; + + if (file.mimetype.startsWith('image')) { + try { + const meta = await sharp(file.path, { + limitInputPixels: false, + }).metadata(); + tempMeta.width = meta.width; + tempMeta.height = meta.height; + } catch (e) { + // Ignore-if file is not image + } + } + const fileName = `${nanoid(18)}${path.extname(originalName)}`; const url = await storageAdapter.fileCreate( @@ -429,6 +455,7 @@ export class PublicDatasService { mimetype: file.mimetype, size: file.size, icon: mimeIcons[path.extname(originalName).slice(1)] || undefined, + ...tempMeta, }); } } @@ -463,15 +490,33 @@ export class PublicDatasService { file?.fileName || file.url.split('/').pop(), )}`; - const attachmentUrl: string | null = await storageAdapter.fileCreateByUrl( + const { url, data } = await storageAdapter.fileCreateByUrl( slash(path.join('nc', 'uploads', ...filePath, fileName)), file.url, ); + const tempMetadata: { + width?: number; + height?: number; + } = {}; + + try { + const metadata = await sharp(data, { + limitInputPixels: true, + }).metadata(); + + if (metadata.width && metadata.height) { + tempMetadata.width = metadata.width; + tempMetadata.height = metadata.height; + } + } catch (e) { + // Might be invalid image - ignore + } + let attachmentPath: string | undefined; // if `attachmentUrl` is null, then it is local attachment - if (!attachmentUrl) { + if (!url) { // then store the attachment path only // url will be constructed in `useAttachmentCell` attachmentPath = `download/${filePath.join('/')}/${fileName}`; @@ -482,18 +527,27 @@ export class PublicDatasService { file.uploadIndex ?? attachments[file.fieldName].length, 0, { - ...(attachmentUrl ? { url: attachmentUrl } : {}), + ...(url ? { url: url } : {}), ...(attachmentPath ? { path: attachmentPath } : {}), title: file.fileName, mimetype: file.mimetype, size: file.size, icon: mimeIcons[path.extname(fileName).slice(1)] || undefined, + ...tempMetadata, }, ); } for (const [column, data] of Object.entries(attachments)) { insertObject[column] = JSON.stringify(data); + + await this.jobsService.add(JobTypes.ThumbnailGenerator, { + attachments: data, + context: { + base_id: RootScopes.ROOT, + workspace_id: RootScopes.ROOT, + }, + }); } return await baseModel.nestedInsert(insertObject, null);