diff --git a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md index 441efc9e4f..66c2fb90a0 100644 --- a/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md +++ b/packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md @@ -32,6 +32,7 @@ For production use cases, it is crucial to set all environment variables marked | -------- | --------- | ----------- | ---------- | | `NC_S3_BUCKET_NAME` | No | The name of the AWS S3 bucket used for the S3 storage plugin. | | | `NC_S3_REGION` | No | The AWS S3 region where the S3 storage plugin bucket is located. | | +| `NC_S3_ENDPOINT` | No | S3 endpoint for S3 storage plugin. | Defaults to `s3..amazonaws.com` | | `NC_S3_ACCESS_KEY` | No | The AWS access key ID required for the S3 storage plugin. | | | `NC_S3_ACCESS_SECRET` | No | The AWS access secret associated with the S3 storage plugin. | | | `NC_ATTACHMENT_FIELD_SIZE` | No | Maximum file size allowed for [attachments](/fields/field-types/custom-types/attachment/) in bytes. | Defaults to `20971520` (20 MiB). | @@ -127,4 +128,3 @@ For production use cases, it is crucial to set all environment variables marked | `AWS_SECRET_ACCESS_KEY` | No | ***Deprecated***. Please use `LITESTREAM_S3_SECRET_ACCESS_KEY` instead. | | | `AWS_BUCKET` | No | ***Deprecated***. Please use `LITESTREAM_S3_BUCKET` instead. | | | `AWS_BUCKET_PATH` | No | ***Deprecated***. Please use `LITESTREAM_S3_PATH` instead. | | - diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index ac057e1e17..d205067f63 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -7100,17 +7100,13 @@ class BaseModelSqlv2 { }).then((r) => (lookedUpAttachment.signedPath = r)), ); } else if (lookedUpAttachment?.url) { - if (lookedUpAttachment.url.includes('.amazonaws.com/')) { - const relativePath = decodeURI( - lookedUpAttachment.url.split('.amazonaws.com/')[1], - ); - promises.push( - PresignedUrl.getSignedUrl({ - path: relativePath, - s3: true, - }).then((r) => (lookedUpAttachment.signedUrl = r)), - ); - } + promises.push( + PresignedUrl.getSignedUrl({ + path: decodeURI( + new URL(lookedUpAttachment.url).pathname, + ), + }).then((r) => (lookedUpAttachment.signedUrl = r)), + ); } } } else { @@ -7121,17 +7117,11 @@ class BaseModelSqlv2 { }).then((r) => (attachment.signedPath = r)), ); } else if (attachment?.url) { - if (attachment.url.includes('.amazonaws.com/')) { - const relativePath = decodeURI( - attachment.url.split('.amazonaws.com/')[1], - ); - promises.push( - PresignedUrl.getSignedUrl({ - path: relativePath, - s3: true, - }).then((r) => (attachment.signedUrl = r)), - ); - } + promises.push( + PresignedUrl.getSignedUrl({ + path: decodeURI(new URL(attachment.url).pathname), + }).then((r) => (attachment.signedUrl = r)), + ); } } } diff --git a/packages/nocodb/src/helpers/NcPluginMgrv2.ts b/packages/nocodb/src/helpers/NcPluginMgrv2.ts index 6abd44b949..b3e647351d 100644 --- a/packages/nocodb/src/helpers/NcPluginMgrv2.ts +++ b/packages/nocodb/src/helpers/NcPluginMgrv2.ts @@ -132,6 +132,7 @@ class NcPluginMgrv2 { input: JSON.stringify({ bucket: process.env.NC_S3_BUCKET_NAME, region: process.env.NC_S3_REGION, + endpoint: process.env.NC_S3_ENDPOINT, access_key: process.env.NC_S3_ACCESS_KEY, access_secret: process.env.NC_S3_ACCESS_SECRET, }), diff --git a/packages/nocodb/src/models/FormView.ts b/packages/nocodb/src/models/FormView.ts index 6dfc767e1a..323d5c5cae 100644 --- a/packages/nocodb/src/models/FormView.ts +++ b/packages/nocodb/src/models/FormView.ts @@ -248,20 +248,12 @@ export default class FormView implements FormViewType { ).then((r) => (formAttachments[key].signedPath = r)), ); } else if (formAttachments[key]?.url) { - if (formAttachments[key].url.includes('.amazonaws.com/')) { - const relativePath = decodeURI( - formAttachments[key].url.split('.amazonaws.com/')[1], - ); - promises.push( - PresignedUrl.getSignedUrl( - { - path: relativePath, - s3: true, - }, - ncMeta, - ).then((r) => (formAttachments[key].signedUrl = r)), - ); - } + promises.push( + PresignedUrl.getSignedUrl( + { path: decodeURI(new URL(formAttachments[key].url).pathname) }, + ncMeta, + ).then((r) => (formAttachments[key].signedUrl = r)), + ); } } await Promise.all(promises); diff --git a/packages/nocodb/src/models/PresignedUrl.ts b/packages/nocodb/src/models/PresignedUrl.ts index a344997263..6e2b3c347e 100644 --- a/packages/nocodb/src/models/PresignedUrl.ts +++ b/packages/nocodb/src/models/PresignedUrl.ts @@ -90,18 +90,13 @@ export default class PresignedUrl { param: { path: string; expireSeconds?: number; - s3?: boolean; filename?: string; }, ncMeta = Noco.ncMeta, ) { - let { path } = param; + let path = param.path.replace(/^\/+/, ''); - const { - expireSeconds = DEFAULT_EXPIRE_SECONDS, - s3 = false, - filename, - } = param; + const { expireSeconds = DEFAULT_EXPIRE_SECONDS, filename } = param; const expireAt = roundExpiry( new Date(new Date().getTime() + expireSeconds * 1000), @@ -130,10 +125,9 @@ export default class PresignedUrl { } } - if (s3) { - // if not present, create a new url - const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta); + const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta); + if (typeof (storageAdapter as any).getSignedUrl === 'function') { tempUrl = await (storageAdapter as any).getSignedUrl( path, expiresInSeconds, diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts index 52f72ddb33..d03f7fe7f9 100644 --- a/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts +++ b/packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts @@ -98,15 +98,11 @@ export class DataExportProcessor { expireSeconds: 3 * 60 * 60, // 3 hours }); } else { - if (url.includes('.amazonaws.com/')) { - const relativePath = decodeURI(url.split('.amazonaws.com/')[1]); - url = await PresignedUrl.getSignedUrl({ - path: relativePath, - filename: `${model.title} (${getViewTitle(view)}).csv`, - s3: true, - expireSeconds: 3 * 60 * 60, // 3 hours - }); - } + url = await PresignedUrl.getSignedUrl({ + path: decodeURI(new URL(url).pathname), + filename: `${model.title} (${getViewTitle(view)}).csv`, + expireSeconds: 3 * 60 * 60, // 3 hours + }); } if (error) { diff --git a/packages/nocodb/src/plugins/s3/S3.ts b/packages/nocodb/src/plugins/s3/S3.ts index 95ab5bf030..86e4ef62cc 100644 --- a/packages/nocodb/src/plugins/s3/S3.ts +++ b/packages/nocodb/src/plugins/s3/S3.ts @@ -5,6 +5,7 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { Upload } from '@aws-sdk/lib-storage'; import axios from 'axios'; import { useAgent } from 'request-filtering-agent'; +import type { S3ClientConfig } from '@aws-sdk/client-s3'; import type { IStorageAdapterV2, XcFile } from 'nc-plugin'; import type { Readable } from 'stream'; import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils'; @@ -128,7 +129,7 @@ export default class S3 implements IStorageAdapterV2 { // s3Options.accessKeyId = process.env.NC_S3_KEY; // s3Options.secretAccessKey = process.env.NC_S3_SECRET; - const s3Options = { + const s3Options: S3ClientConfig = { region: this.input.region, credentials: { accessKeyId: this.input.access_key, @@ -136,6 +137,10 @@ export default class S3 implements IStorageAdapterV2 { }, }; + if (this.input.endpoint) { + s3Options.endpoint = this.input.endpoint; + } + this.s3Client = new S3Client(s3Options); } @@ -168,7 +173,10 @@ export default class S3 implements IStorageAdapterV2 { const data = await upload.done(); if (data) { - return `https://${this.input.bucket}.s3.${this.input.region}.amazonaws.com/${uploadParams.Key}`; + const endpoint = this.input.endpoint + ? new URL(this.input.endpoint).host + : `s3.${this.input.region}.amazonaws.com`; + return `https://${this.input.bucket}.${endpoint}/${uploadParams.Key}`; } else { throw new Error('Upload failed or no data returned.'); } diff --git a/packages/nocodb/src/plugins/s3/index.ts b/packages/nocodb/src/plugins/s3/index.ts index a5ca6d75a9..517e2bed76 100644 --- a/packages/nocodb/src/plugins/s3/index.ts +++ b/packages/nocodb/src/plugins/s3/index.ts @@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin'; const config: XcPluginConfig = { builder: S3Plugin, title: 'S3', - version: '0.0.1', + version: '0.0.2', logo: 'plugins/s3.png', description: 'Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.', @@ -26,6 +26,13 @@ const config: XcPluginConfig = { type: XcType.SingleLineText, required: true, }, + { + key: 'endpoint', + label: 'Endpoint', + placeholder: 'Endpoint', + type: XcType.SingleLineText, + required: false, + }, { key: 'access_key', label: 'Access Key', diff --git a/packages/nocodb/src/services/attachments.service.ts b/packages/nocodb/src/services/attachments.service.ts index 263325b797..34eeca21f1 100644 --- a/packages/nocodb/src/services/attachments.service.ts +++ b/packages/nocodb/src/services/attachments.service.ts @@ -85,16 +85,9 @@ export class AttachmentsService { path: attachment.path.replace(/^download\//, ''), }); } else { - if (attachment.url.includes('.amazonaws.com/')) { - const relativePath = decodeURI( - attachment.url.split('.amazonaws.com/')[1], - ); - - attachment.signedUrl = await PresignedUrl.getSignedUrl({ - path: relativePath, - s3: true, - }); - } + attachment.signedUrl = await PresignedUrl.getSignedUrl({ + path: decodeURI(new URL(attachment.url).pathname), + }); } attachments.push(attachment);