feat: add S3 endpoint input (#8886)

* feat: add S3 endpoint input

* fix: generate signed urls for non-aws attachments

packages/noco-docs/docs/020.getting-started/050.self-hosted/020.environment-variables.md

@@ -32,6 +32,7 @@ For production use cases, it is crucial to set all environment variables marked
 | -------- | --------- | ----------- | ---------- |
 | `NC_S3_BUCKET_NAME` | No | The name of the AWS S3 bucket used for the S3 storage plugin. | |
 | `NC_S3_REGION` | No | The AWS S3 region where the S3 storage plugin bucket is located. | |
+| `NC_S3_ENDPOINT` | No | S3 endpoint for S3 storage plugin. |  Defaults to `s3.<region>.amazonaws.com` |
 | `NC_S3_ACCESS_KEY` | No | The AWS access key ID required for the S3 storage plugin. | |
 | `NC_S3_ACCESS_SECRET` | No | The AWS access secret associated with the S3 storage plugin. | |
 | `NC_ATTACHMENT_FIELD_SIZE` | No | Maximum file size allowed for [attachments](/fields/field-types/custom-types/attachment/) in bytes. | Defaults to `20971520` (20 MiB). |
@@ -127,4 +128,3 @@ For production use cases, it is crucial to set all environment variables marked
 | `AWS_SECRET_ACCESS_KEY` | No | ***Deprecated***. Please use `LITESTREAM_S3_SECRET_ACCESS_KEY` instead. | |
 | `AWS_BUCKET` | No | ***Deprecated***. Please use `LITESTREAM_S3_BUCKET` instead. | |
 | `AWS_BUCKET_PATH` | No | ***Deprecated***. Please use `LITESTREAM_S3_PATH` instead. | |
-

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -7100,17 +7100,13 @@ class BaseModelSqlv2 {
                       }).then((r) => (lookedUpAttachment.signedPath = r)),
                     );
                   } else if (lookedUpAttachment?.url) {
-                    if (lookedUpAttachment.url.includes('.amazonaws.com/')) {
-                      const relativePath = decodeURI(
-                        lookedUpAttachment.url.split('.amazonaws.com/')[1],
-                      );
-                      promises.push(
-                        PresignedUrl.getSignedUrl({
-                          path: relativePath,
-                          s3: true,
-                        }).then((r) => (lookedUpAttachment.signedUrl = r)),
-                      );
-                    }
+                    promises.push(
+                      PresignedUrl.getSignedUrl({
+                        path: decodeURI(
+                          new URL(lookedUpAttachment.url).pathname,
+                        ),
+                      }).then((r) => (lookedUpAttachment.signedUrl = r)),
+                    );
                   }
                 }
               } else {
@@ -7121,17 +7117,11 @@ class BaseModelSqlv2 {
                     }).then((r) => (attachment.signedPath = r)),
                   );
                 } else if (attachment?.url) {
-                  if (attachment.url.includes('.amazonaws.com/')) {
-                    const relativePath = decodeURI(
-                      attachment.url.split('.amazonaws.com/')[1],
-                    );
-                    promises.push(
-                      PresignedUrl.getSignedUrl({
-                        path: relativePath,
-                        s3: true,
-                      }).then((r) => (attachment.signedUrl = r)),
-                    );
-                  }
+                  promises.push(
+                    PresignedUrl.getSignedUrl({
+                      path: decodeURI(new URL(attachment.url).pathname),
+                    }).then((r) => (attachment.signedUrl = r)),
+                  );
                 }
               }
             }

packages/nocodb/src/helpers/NcPluginMgrv2.ts

@@ -132,6 +132,7 @@ class NcPluginMgrv2 {
         input: JSON.stringify({
           bucket: process.env.NC_S3_BUCKET_NAME,
           region: process.env.NC_S3_REGION,
+          endpoint: process.env.NC_S3_ENDPOINT,
           access_key: process.env.NC_S3_ACCESS_KEY,
           access_secret: process.env.NC_S3_ACCESS_SECRET,
         }),

packages/nocodb/src/models/FormView.ts

@@ -248,20 +248,12 @@ export default class FormView implements FormViewType {
               ).then((r) => (formAttachments[key].signedPath = r)),
             );
           } else if (formAttachments[key]?.url) {
-            if (formAttachments[key].url.includes('.amazonaws.com/')) {
-              const relativePath = decodeURI(
-                formAttachments[key].url.split('.amazonaws.com/')[1],
-              );
-              promises.push(
-                PresignedUrl.getSignedUrl(
-                  {
-                    path: relativePath,
-                    s3: true,
-                  },
-                  ncMeta,
-                ).then((r) => (formAttachments[key].signedUrl = r)),
-              );
-            }
+            promises.push(
+              PresignedUrl.getSignedUrl(
+                { path: decodeURI(new URL(formAttachments[key].url).pathname) },
+                ncMeta,
+              ).then((r) => (formAttachments[key].signedUrl = r)),
+            );
           }
         }
         await Promise.all(promises);

packages/nocodb/src/models/PresignedUrl.ts

@@ -90,18 +90,13 @@ export default class PresignedUrl {
     param: {
       path: string;
       expireSeconds?: number;
-      s3?: boolean;
       filename?: string;
     },
     ncMeta = Noco.ncMeta,
   ) {
-    let { path } = param;
+    let path = param.path.replace(/^\/+/, '');
 
-    const {
-      expireSeconds = DEFAULT_EXPIRE_SECONDS,
-      s3 = false,
-      filename,
-    } = param;
+    const { expireSeconds = DEFAULT_EXPIRE_SECONDS, filename } = param;
 
     const expireAt = roundExpiry(
       new Date(new Date().getTime() + expireSeconds * 1000),
@@ -130,10 +125,9 @@ export default class PresignedUrl {
       }
     }
 
-    if (s3) {
-      // if not present, create a new url
-      const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta);
+    const storageAdapter = await NcPluginMgrv2.storageAdapter(ncMeta);
 
+    if (typeof (storageAdapter as any).getSignedUrl === 'function') {
       tempUrl = await (storageAdapter as any).getSignedUrl(
         path,
         expiresInSeconds,

packages/nocodb/src/modules/jobs/jobs/data-export/data-export.processor.ts

@@ -98,15 +98,11 @@ export class DataExportProcessor {
           expireSeconds: 3 * 60 * 60, // 3 hours
         });
       } else {
-        if (url.includes('.amazonaws.com/')) {
-          const relativePath = decodeURI(url.split('.amazonaws.com/')[1]);
-          url = await PresignedUrl.getSignedUrl({
-            path: relativePath,
-            filename: `${model.title} (${getViewTitle(view)}).csv`,
-            s3: true,
-            expireSeconds: 3 * 60 * 60, // 3 hours
-          });
-        }
+        url = await PresignedUrl.getSignedUrl({
+          path: decodeURI(new URL(url).pathname),
+          filename: `${model.title} (${getViewTitle(view)}).csv`,
+          expireSeconds: 3 * 60 * 60, // 3 hours
+        });
       }
 
       if (error) {

packages/nocodb/src/plugins/s3/S3.ts

@@ -5,6 +5,7 @@ import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Upload } from '@aws-sdk/lib-storage';
 import axios from 'axios';
 import { useAgent } from 'request-filtering-agent';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -128,7 +129,7 @@ export default class S3 implements IStorageAdapterV2 {
     // s3Options.accessKeyId = process.env.NC_S3_KEY;
     // s3Options.secretAccessKey = process.env.NC_S3_SECRET;
 
-    const s3Options = {
+    const s3Options: S3ClientConfig = {
       region: this.input.region,
       credentials: {
         accessKeyId: this.input.access_key,
@@ -136,6 +137,10 @@ export default class S3 implements IStorageAdapterV2 {
       },
     };
 
+    if (this.input.endpoint) {
+      s3Options.endpoint = this.input.endpoint;
+    }
+
     this.s3Client = new S3Client(s3Options);
   }
 
@@ -168,7 +173,10 @@ export default class S3 implements IStorageAdapterV2 {
       const data = await upload.done();
 
       if (data) {
-        return `https://${this.input.bucket}.s3.${this.input.region}.amazonaws.com/${uploadParams.Key}`;
+        const endpoint = this.input.endpoint
+          ? new URL(this.input.endpoint).host
+          : `s3.${this.input.region}.amazonaws.com`;
+        return `https://${this.input.bucket}.${endpoint}/${uploadParams.Key}`;
       } else {
         throw new Error('Upload failed or no data returned.');
       }

packages/nocodb/src/plugins/s3/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: S3Plugin,
   title: 'S3',
-  version: '0.0.1',
+  version: '0.0.2',
   logo: 'plugins/s3.png',
   description:
     'Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.',
@@ -26,6 +26,13 @@ const config: XcPluginConfig = {
         type: XcType.SingleLineText,
         required: true,
       },
+      {
+        key: 'endpoint',
+        label: 'Endpoint',
+        placeholder: 'Endpoint',
+        type: XcType.SingleLineText,
+        required: false,
+      },
       {
         key: 'access_key',
         label: 'Access Key',

packages/nocodb/src/services/attachments.service.ts

@@ -85,16 +85,9 @@ export class AttachmentsService {
               path: attachment.path.replace(/^download\//, ''),
             });
           } else {
-            if (attachment.url.includes('.amazonaws.com/')) {
-              const relativePath = decodeURI(
-                attachment.url.split('.amazonaws.com/')[1],
-              );
-
-              attachment.signedUrl = await PresignedUrl.getSignedUrl({
-                path: relativePath,
-                s3: true,
-              });
-            }
+            attachment.signedUrl = await PresignedUrl.getSignedUrl({
+              path: decodeURI(new URL(attachment.url).pathname),
+            });
           }
 
           attachments.push(attachment);