Browse Source

fix: handle data url using stream

pull/9722/head
mertmit 1 week ago
parent
commit
e75556c30d
  1. 5
      packages/nocodb/src/plugins/GenericS3/GenericS3.ts
  2. 18
      packages/nocodb/src/plugins/storage/Local.ts
  3. 49
      packages/nocodb/src/services/attachments.service.ts
  4. 8
      packages/nocodb/src/types/nc-plugin/lib/IStorageAdapterV2.ts

5
packages/nocodb/src/plugins/GenericS3/GenericS3.ts

@ -104,7 +104,10 @@ export default class GenericS3 implements IStorageAdapterV2 {
options?: { options?: {
mimetype?: string; mimetype?: string;
}, },
): Promise<void> { ): Promise<{
url: string | null;
data: any;
}> {
try { try {
const streamError = new Promise<void>((_, reject) => { const streamError = new Promise<void>((_, reject) => {
stream.on('error', (err) => { stream.on('error', (err) => {

18
packages/nocodb/src/plugins/storage/Local.ts

@ -70,13 +70,27 @@ export default class Local implements IStorageAdapterV2 {
public async fileCreateByStream( public async fileCreateByStream(
key: string, key: string,
stream: Readable, stream: Readable,
): Promise<void> { ): Promise<{
url: string | null;
data: any;
}> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const destPath = validateAndNormaliseLocalPath(key); const destPath = validateAndNormaliseLocalPath(key);
try { try {
mkdirp(path.dirname(destPath)).then(() => { mkdirp(path.dirname(destPath)).then(() => {
const writableStream = fs.createWriteStream(destPath); const writableStream = fs.createWriteStream(destPath);
writableStream.on('finish', () => resolve()); writableStream.on('finish', () => {
this.fileRead(destPath)
.then((data) => {
resolve({
url: null,
data,
});
})
.catch((e) => {
reject(e);
});
});
writableStream.on('error', (err) => reject(err)); writableStream.on('error', (err) => reject(err));
stream.pipe(writableStream); stream.pipe(writableStream);
}); });

49
packages/nocodb/src/services/attachments.service.ts

@ -1,5 +1,6 @@
import path from 'path'; import path from 'path';
import Url from 'url'; import Url from 'url';
import { Readable } from 'stream';
import { AppEvents, PublicAttachmentScope } from 'nocodb-sdk'; import { AppEvents, PublicAttachmentScope } from 'nocodb-sdk';
import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common'; import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common';
import { nanoid } from 'nanoid'; import { nanoid } from 'nanoid';
@ -280,11 +281,23 @@ export class AttachmentsService {
size, size,
finalUrl = url; finalUrl = url;
let base64TempStream: Readable;
if (!url.startsWith('data:')) { if (!url.startsWith('data:')) {
response = await axios.head(url, { maxRedirects: 5 }); response = await axios.head(url, { maxRedirects: 5 });
mimeType = response.headers['content-type']?.split(';')[0]; mimeType = response.headers['content-type']?.split(';')[0];
size = response.headers['content-length']; size = response.headers['content-length'];
finalUrl = response.request.res.responseUrl; finalUrl = response.request.res.responseUrl;
} else {
const matches = url.match(/^data:(.+);base64,(.+)$/);
if (!matches) throw new Error('Invalid data URL format.');
// Extract MIME type and base64 data
const [, _mimeType, base64Data] = matches;
mimeType = _mimeType;
size = Buffer.byteLength(base64Data, 'base64');
base64TempStream = Readable.from(Buffer.from(base64Data, 'base64'));
} }
const parsedUrl = Url.parse(finalUrl, true); const parsedUrl = Url.parse(finalUrl, true);
@ -303,17 +316,33 @@ export class AttachmentsService {
mimeType = mime.getType(path.extname(fileNameWithExt).slice(1)); mimeType = mime.getType(path.extname(fileNameWithExt).slice(1));
} }
const { url: attachmentUrl, data: file } = let attachmentUrl, file;
await storageAdapter.fileCreateByUrl(
slash(path.join(fileDestPath, fileName)), if (!base64TempStream) {
finalUrl, const { url: _attachmentUrl, data: _file } =
{ await storageAdapter.fileCreateByUrl(
fetchOptions: { slash(path.join(fileDestPath, fileName)),
// The sharp requires image to be passed as buffer.); finalUrl,
buffer: mimeType.includes('image'), {
fetchOptions: {
// The sharp requires image to be passed as buffer.);
buffer: mimeType.includes('image'),
},
}, },
}, );
);
attachmentUrl = _attachmentUrl;
file = _file;
} else {
const { url: _attachmentUrl, data: _file } =
(await storageAdapter.fileCreateByStream(
slash(path.join(fileDestPath, fileName)),
base64TempStream,
)) as any;
attachmentUrl = _attachmentUrl;
file = _file;
}
const tempMetadata: { const tempMetadata: {
width?: number; width?: number;

8
packages/nocodb/src/types/nc-plugin/lib/IStorageAdapterV2.ts

@ -30,7 +30,13 @@ export default interface IStorageAdapterV2<
url: string, url: string,
options?: FileCreateByUrlOptions, options?: FileCreateByUrlOptions,
): Promise<any>; ): Promise<any>;
fileCreateByStream(destPath: string, readStream: Readable): Promise<void>; fileCreateByStream(
destPath: string,
readStream: Readable,
): Promise<{
url: string | null;
data: any;
}>;
fileReadByStream( fileReadByStream(
key: string, key: string,
options?: { encoding?: string }, options?: { encoding?: string },

Loading…
Cancel
Save