chore: migrate to aws sdk v3 (#9104)

* feat: backblaze migrate

* feat: scaleway migrate

* feat: digital ocean

* chore: migrate to aws-sdk v3

* fix: refactor storagePlugins

* fix: handle error inside callback

* fix: return data as Buffer

* fix: backblaze handle

* feat: acl support for storage plugins

* feat: minio refactor

* feat: R2 plugin

* fix: lock file

* fix: use buffer only for images

* fix: update fileCreateByUrl

* fix: upocloud case

* fix: upocloud case

* chore: sync dependencies

packages/nocodb/package.json

@@ -45,9 +45,13 @@
     "docker:build": "EE=\"true-xc-test\" webpack  --config docker/webpack.config.js"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.504.0",
+    "@aws-sdk/client-kafka": "^3.620.0",
-    "@aws-sdk/lib-storage": "^3.504.0",
+    "@aws-sdk/client-kinesis": "^3.620.0",
-    "@aws-sdk/s3-request-presigner": "^3.504.0",
+    "@aws-sdk/client-s3": "^3.620.0",
+    "@aws-sdk/client-ses": "^3.620.0",
+    "@aws-sdk/client-sns": "^3.620.0",
+    "@aws-sdk/lib-storage": "^3.620.0",
+    "@aws-sdk/s3-request-presigner": "^3.620.0",
     "@google-cloud/storage": "^7.7.0",
     "@jm18457/kafkajs-msk-iam-authentication-mechanism": "^3.1.2",
     "@nestjs/bull": "^10.0.1",
@@ -112,7 +116,7 @@
     "lodash": "^4.17.21",
     "mailersend": "^1.5.0",
     "marked": "^4.3.0",
-    "minio": "^7.1.3",
+    "minio": "^8.0.1",
     "mkdirp": "^2.1.6",
     "mssql": "^10.0.2",
     "multer": "^1.4.5-lts.1",
@@ -120,7 +124,7 @@
     "nanoid": "^3.3.7",
     "nc-help": "0.3.1",
     "nc-lib-gui": "0.251.3",
-    "nc-plugin": "^0.1.3",
+    "nc-plugin": "^0.1.6",
     "nestjs-throttler-storage-redis": "^0.4.4",
     "nocodb-sdk": "workspace:^",
     "nodemailer": "^6.9.13",

packages/nocodb/src/helpers/NcPluginMgrv2.ts

@@ -28,6 +28,7 @@ import TwilioWhatsappPluginConfig from '~/plugins/twilioWhatsapp';
 import UpcloudPluginConfig from '~/plugins/upcloud';
 import VultrPluginConfig from '~/plugins/vultr';
 import SESPluginConfig from '~/plugins/ses';
+import R2PluginConfig from '~/plugins/r2';
 import Noco from '~/Noco';
 import Local from '~/plugins/storage/Local';
 import { MetaTable, RootScopes } from '~/utils/globals';
@@ -53,6 +54,7 @@ const defaultPlugins = [
   MailerSendConfig,
   ScalewayPluginConfig,
   SESPluginConfig,
+  R2PluginConfig,
 ];
 
 class NcPluginMgrv2 {

packages/nocodb/src/plugins/GenericS3/GenericS3.ts

@@ -0,0 +1,199 @@
+import fs from 'fs';
+import { promisify } from 'util';
+import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
+import {
+  GetObjectCommand,
+  type PutObjectCommandInput,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+import { Upload } from '@aws-sdk/lib-storage';
+import type { PutObjectRequest, S3 as S3Client } from '@aws-sdk/client-s3';
+import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+import type { Readable } from 'stream';
+import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+
+interface GenerocObjectStorageInput {
+  bucket: string;
+  region?: string;
+  access_key: string;
+  access_secret: string;
+}
+
+export default class GenericS3 implements IStorageAdapterV2 {
+  protected s3Client: S3Client;
+  protected input: GenerocObjectStorageInput;
+
+  constructor(input: unknown) {
+    this.input = input as GenerocObjectStorageInput;
+  }
+
+  protected get defaultParams() {
+    return {
+      Bucket: this.input.bucket,
+    };
+  }
+
+  public async init(): Promise<any> {
+    // Placeholder, should be initalized in child class
+  }
+
+  public async test(): Promise<boolean> {
+    try {
+      const tempFile = generateTempFilePath();
+      const createStream = fs.createWriteStream(tempFile);
+      await waitForStreamClose(createStream);
+      await this.fileCreate('nc-test-file.txt', {
+        path: tempFile,
+        mimetype: 'text/plain',
+        originalname: 'temp.txt',
+        size: '',
+      });
+      await promisify(fs.unlink)(tempFile);
+      return true;
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  public async fileRead(key: string): Promise<any> {
+    const command = new GetObjectCommand({
+      Key: key,
+      Bucket: this.input.bucket,
+    });
+
+    const { Body } = await this.s3Client.send(command);
+
+    const fileStream = Body as Readable;
+
+    return new Promise((resolve, reject) => {
+      const chunks: any[] = [];
+      fileStream.on('data', (chunk) => {
+        chunks.push(chunk);
+      });
+
+      fileStream.on('end', () => {
+        const buffer = Buffer.concat(chunks);
+        resolve(buffer);
+      });
+
+      fileStream.on('error', (err) => {
+        reject(err);
+      });
+    });
+  }
+
+  async fileCreate(key: string, file: XcFile): Promise<any> {
+    const fileStream = fs.createReadStream(file.path);
+
+    return this.fileCreateByStream(key, fileStream, {
+      mimetype: file?.mimetype,
+    });
+  }
+
+  async fileCreateByStream(
+    key: string,
+    stream: Readable,
+    options?: {
+      mimetype?: string;
+    },
+  ): Promise<void> {
+    try {
+      const streamError = new Promise<void>((_, reject) => {
+        stream.on('error', (err) => {
+          reject(err);
+        });
+      });
+
+      const uploadParams = {
+        ...this.defaultParams,
+        Body: stream,
+        Key: key,
+        ContentType: options?.mimetype || 'application/octet-stream',
+      };
+
+      const upload = this.upload(uploadParams);
+
+      return await Promise.race([upload, streamError]);
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  async fileCreateByUrl(
+    key: string,
+    url: string,
+    { fetchOptions: { buffer } = { buffer: false } },
+  ): Promise<any> {
+    try {
+      const response = await axios.get(url, {
+        httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        responseType: buffer ? 'arraybuffer' : 'stream',
+      });
+      const uploadParams: PutObjectRequest = {
+        ...this.defaultParams,
+        Body: response.data,
+        Key: key,
+        ContentType: response.headers['content-type'],
+      };
+
+      const data = await this.upload(uploadParams);
+      return {
+        url: data,
+        data: response.data,
+      };
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  public async getSignedUrl(
+    key,
+    expiresInSeconds = 7200,
+    pathParameters?: { [key: string]: string },
+  ) {
+    const command = new GetObjectCommand({
+      Key: key,
+      Bucket: this.input.bucket,
+      ...pathParameters,
+    });
+    return getSignedUrl(this.s3Client, command, {
+      expiresIn: expiresInSeconds,
+    });
+  }
+
+  protected async upload(uploadParams: PutObjectCommandInput): Promise<any> {
+    try {
+      const upload = new Upload({
+        client: this.s3Client,
+        params: {
+          ACL: 'public-read',
+          ...uploadParams,
+        },
+      });
+
+      const data = await upload.done();
+
+      return data.Location;
+    } catch (error) {
+      console.error('Error uploading file', error);
+      throw error;
+    }
+  }
+
+  // TODO - implement
+  fileReadByStream(_key: string): Promise<Readable> {
+    return Promise.resolve(undefined);
+  }
+
+  // TODO - implement
+  getDirectoryList(_path: string): Promise<string[]> {
+    return Promise.resolve(undefined);
+  }
+
+  public async fileDelete(_path: string): Promise<any> {
+    return Promise.resolve(undefined);
+  }
+}

packages/nocodb/src/plugins/backblaze/Backblaze.ts

@@ -1,101 +1,86 @@
-import fs from 'fs';
+import { Upload } from '@aws-sdk/lib-storage';
-import { promisify } from 'util';
+import { GetObjectCommand, S3 as S3Client } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
-import axios from 'axios';
+import type { PutObjectCommandInput, S3ClientConfig } from '@aws-sdk/client-s3';
-import { useAgent } from 'request-filtering-agent';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import type { Readable } from 'stream';
+
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+interface BackblazeObjectStorageInput {
+  bucket: string;
+  region: string;
+  access_key: string;
+  access_secret: string;
+  acl?: string;
+}
 
-export default class Backblaze implements IStorageAdapterV2 {
+export default class Backblaze extends GenericS3 implements IStorageAdapterV2 {
-  private s3Client: AWS.S3;
+  protected input: BackblazeObjectStorageInput;
-  private input: any;
 
-  constructor(input: any) {
+  constructor(input: unknown) {
-    this.input = input;
+    super(input as BackblazeObjectStorageInput);
   }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
+  protected get defaultParams() {
-    const fileStream = fs.createReadStream(file.path);
+    return {
-
+      Bucket: this.input.bucket,
-    return this.fileCreateByStream(key, fileStream, {
+      ACL: this.input?.acl || 'public-read',
-      mimetype: file?.mimetype,
+    };
-    });
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  public async init(): Promise<any> {
-    const uploadParams: any = {
+    const s3Options: S3ClientConfig = {
-      ACL: 'public-read',
+      region: this.patchRegion(this.input.region),
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+      endpoint: `https://s3.${this.patchRegion(
+        this.input.region,
+      )}.backblazeb2.com`,
     };
 
-    return new Promise((resolve, reject) => {
+    this.s3Client = new S3Client(s3Options);
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          // call S3 to retrieve upload file to specified bucket
-          this.s3Client.upload(uploadParams, (err1, data) => {
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
   }
-            if (data) {
+
-              resolve({
+  public async getSignedUrl(
-                url: data.Location,
+    key,
-                data: response.data,
+    expiresInSeconds = 7200,
-              });
+    pathParameters?: { [key: string]: string },
+  ) {
+    let tempKey = key;
+    if (
+      tempKey.startsWith(`${this.input.bucket}/nc/uploads`) ||
+      tempKey.startsWith(`${this.input.bucket}/nc/thumbnails`)
+    ) {
+      tempKey = tempKey.replace(`${this.input.bucket}/`, '');
     }
+
+    const command = new GetObjectCommand({
+      Key: tempKey,
+      Bucket: this.input.bucket,
+      ...pathParameters,
     });
-        })
+    return getSignedUrl(this.s3Client, command, {
-        .catch((error) => {
+      expiresIn: expiresInSeconds,
-          reject(error);
-        });
     });
   }
 
-  async fileCreateByStream(
+  protected async upload(uploadParams: PutObjectCommandInput): Promise<any> {
-    key: string,
+    try {
-    stream: Readable,
+      const upload = new Upload({
-    options?: {
+        client: this.s3Client,
-      mimetype?: string;
+        params: uploadParams,
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Body: stream,
-      Key: key,
-      ContentType: options?.mimetype || 'application/octet-stream',
-    };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
       });
-  }
 
-  // TODO - implement
+      const data = await upload.done();
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
 
-  // TODO - implement
+      if (data) {
-  getDirectoryList(_path: string): Promise<string[]> {
+        return `https://${this.input.bucket}.s3.${this.input.region}.backblazeb2.com/${uploadParams.Key}`;
-    return Promise.resolve(undefined);
+      }
+    } catch (error) {
+      console.error('Error uploading file', error);
+      throw error;
+    }
   }
 
   patchRegion(region: string): string {
@@ -107,56 +92,4 @@ export default class Backblaze implements IStorageAdapterV2 {
     }
     return region;
   }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
-  }
-
-  public async init(): Promise<any> {
-    const s3Options: any = {
-      params: { Bucket: this.input.bucket },
-      region: this.patchRegion(this.input.region),
-    };
-
-    s3Options.accessKeyId = this.input.access_key;
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(
-      `s3.${s3Options.region}.backblazeb2.com`,
-    );
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
-  }
 }

packages/nocodb/src/plugins/backblaze/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: BackblazePlugin,
   title: 'Backblaze B2',
-  version: '0.0.2',
+  version: '0.0.3',
   logo: 'plugins/backblaze.jpeg',
   tags: 'Storage',
   description:
@@ -41,6 +41,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/gcs/Gcs.ts

@@ -3,73 +3,29 @@ import { promisify } from 'util';
 import { Storage } from '@google-cloud/storage';
 import axios from 'axios';
 import { useAgent } from 'request-filtering-agent';
+import type { GetSignedUrlConfig, StorageOptions } from '@google-cloud/storage';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
-import type { StorageOptions } from '@google-cloud/storage';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
 
+interface GoogleCloudStorageInput {
+  client_email: string;
+  private_key: string;
+  bucket: string;
+  project_id: string;
+}
+
 export default class Gcs implements IStorageAdapterV2 {
   private storageClient: Storage;
   private bucketName: string;
-  private input: any;
+  private input: GoogleCloudStorageInput;
-
-  constructor(input: any) {
-    this.input = input;
-  }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
+  constructor(input: unknown) {
-    const uploadResponse = await this.storageClient
+    this.input = input as GoogleCloudStorageInput;
-      .bucket(this.bucketName)
-      .upload(file.path, {
-        destination: key,
-        contentType: file?.mimetype || 'application/octet-stream',
-        // Support for HTTP requests made with `Accept-Encoding: gzip`
-        gzip: true,
-        // By setting the option `destination`, you can change the name of the
-        // object you are uploading to a bucket.
-        metadata: {
-          // Enable long-lived HTTP caching headers
-          // Use only if the contents of the file will never change
-          // (If the contents will change, use cacheControl: 'no-cache')
-          cacheControl: 'public, max-age=31536000',
-        },
-      });
-
-    return uploadResponse[0].publicUrl();
-  }
-
-  fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      const file = this.storageClient.bucket(this.bucketName).file(key);
-      // Check for existence, since gcloud-node seemed to be caching the result
-      file.exists((err, exists) => {
-        if (exists) {
-          file.download((downerr, data) => {
-            if (err) {
-              return reject(downerr);
-            }
-            return resolve(data);
-          });
-        } else {
-          reject(err);
-        }
-      });
-    });
   }
 
   public async init(): Promise<any> {
     const options: StorageOptions = {};
-
-    // options.credentials = {
-    //   client_email: process.env.NC_GCS_CLIENT_EMAIL,
-    //   private_key: process.env.NC_GCS_PRIVATE_KEY
-    // }
-    //
-    // this.bucketName = process.env.NC_GCS_BUCKET;
     options.credentials = {
       client_email: this.input.client_email,
       // replace \n with real line breaks to avoid
@@ -81,9 +37,7 @@ export default class Gcs implements IStorageAdapterV2 {
     if (this.input.project_id) {
       options.projectId = this.input.project_id;
     }
-
     this.bucketName = this.input.bucket;
-
     this.storageClient = new Storage(options);
   }
 
@@ -105,29 +59,40 @@ export default class Gcs implements IStorageAdapterV2 {
     }
   }
 
-  fileCreateByUrl(destPath: string, url: string): Promise<any> {
+  public fileRead(key: string): Promise<any> {
     return new Promise((resolve, reject) => {
-      axios
+      const file = this.storageClient.bucket(this.bucketName).file(key);
-        .get(url, {
+      // Check for existence, since gcloud-node seemed to be caching the result
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+      file.exists((err, exists) => {
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        if (exists) {
-          // TODO - use stream instead of buffer
+          file.download((downerr, data) => {
-          responseType: 'arraybuffer',
+            if (err) {
-        })
+              return reject(downerr);
-        .then((response) => {
+            }
-          this.storageClient
+            return resolve(data);
-            .bucket(this.bucketName)
+          });
-            .file(destPath)
+        } else {
-            .save(response.data)
+          reject(err);
-            .then((res) => resolve({ url: res, data: response.data }))
+        }
-            .catch(reject);
-        })
-        .catch((error) => {
-          reject(error);
       });
     });
   }
 
+  async fileCreate(key: string, file: XcFile): Promise<any> {
+    const uploadResponse = await this.storageClient
+      .bucket(this.bucketName)
+      .upload(file.path, {
+        destination: key,
+        contentType: file?.mimetype || 'application/octet-stream',
+        gzip: true,
+        metadata: {
+          cacheControl: 'public, max-age=31536000',
+        },
+      });
+
+    return uploadResponse[0].publicUrl();
+  }
+
   async fileCreateByStream(
     key: string,
     stream: Readable,
@@ -155,6 +120,36 @@ export default class Gcs implements IStorageAdapterV2 {
     return uploadResponse[0].publicUrl();
   }
 
+  async fileCreateByUrl(
+    destPath: string,
+    url: string,
+    { fetchOptions: { buffer } = { buffer: false } },
+  ): Promise<any> {
+    return new Promise((resolve, reject) => {
+      axios
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: buffer ? 'arraybuffer' : 'stream',
+        })
+        .then((response) => {
+          this.storageClient
+            .bucket(this.bucketName)
+            .file(destPath)
+            .save(response.data)
+            .then((res) => resolve({ url: res, data: response.data }))
+            .catch(reject);
+        })
+        .catch((error) => {
+          reject(error);
+        });
+    });
+  }
+
+  fileDelete(_path: string): Promise<any> {
+    return Promise.resolve(undefined);
+  }
+
   // TODO - implement
   fileReadByStream(_key: string): Promise<Readable> {
     return Promise.resolve(undefined);
@@ -164,4 +159,24 @@ export default class Gcs implements IStorageAdapterV2 {
   getDirectoryList(_path: string): Promise<string[]> {
     return Promise.resolve(undefined);
   }
+
+  public async getSignedUrl(
+    key,
+    expiresInSeconds = 7200,
+    pathParameters?: { [key: string]: string },
+  ) {
+    const options: GetSignedUrlConfig = {
+      version: 'v4',
+      action: 'read',
+      expires: Date.now() + expiresInSeconds * 1000,
+      extensionHeaders: pathParameters,
+    };
+
+    const [url] = await this.storageClient
+      .bucket(this.bucketName)
+      .file(key)
+      .getSignedUrl(options);
+
+    return url;
+  }
 }

packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts

@@ -1,151 +1,42 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import axios from 'axios';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { useAgent } from 'request-filtering-agent';
+
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+interface LinodeObjectStorageInput {
-import type { Readable } from 'stream';
+  bucket: string;
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  region: string;
-
+  access_key: string;
-export default class LinodeObjectStorage implements IStorageAdapterV2 {
+  access_secret: string;
-  private s3Client: AWS.S3;
+  acl?: string;
-  private input: any;
+}
-
-  constructor(input: any) {
-    this.input = input;
-  }
-
-  async fileCreate(key: string, file: XcFile): Promise<any> {
-    const fileStream = fs.createReadStream(file.path);
-
-    return this.fileCreateByStream(key, fileStream, {
-      mimetype: file?.mimetype,
-    });
-  }
-
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-    };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
 
-          // call S3 to retrieve upload file to specified bucket
+export default class LinodeObjectStorage
-          this.s3Client.upload(uploadParams, (err1, data) => {
+  extends GenericS3
-            if (err1) {
+  implements IStorageAdapterV2
-              console.log('Error', err1);
+{
-              reject(err1);
+  protected input: LinodeObjectStorageInput;
-            }
+  constructor(input: unknown) {
-            if (data) {
+    super(input as LinodeObjectStorageInput);
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
   }
 
-  public async fileCreateByStream(
+  protected get defaultParams() {
-    key: string,
+    return {
-    stream: Readable,
+      Bucket: this.input.bucket,
-    options?: {
+      ACL: this.input?.acl || 'public-read',
-      mimetype?: string;
-    },
-  ): Promise<any> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Body: stream,
-      Key: key,
-      ContentType: options?.mimetype || 'application/octet-stream',
     };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    const s3Options: any = {
+    const s3Options: S3ClientConfig = {
-      params: { Bucket: this.input.bucket },
       region: this.input.region,
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+      endpoint: `https://${this.input.region}.linodeobjects.com`,
     };
 
-    s3Options.accessKeyId = this.input.access_key;
+    this.s3Client = new S3Client(s3Options);
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(
-      `${this.input.region}.linodeobjects.com`,
-    );
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 }

packages/nocodb/src/plugins/linode/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: LinodeObjectStoragePlugin,
   title: 'Linode Object Storage',
-  version: '0.0.1',
+  version: '0.0.2',
   logo: 'plugins/linode.svg',
   tags: 'Storage',
   description:
@@ -41,6 +41,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/mino/Minio.ts

@@ -1,141 +1,182 @@
 import fs from 'fs';
-import { promisify } from 'util';
+import { Readable } from 'stream';
 import { Client as MinioClient } from 'minio';
 import axios from 'axios';
 import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
-import type { Readable } from 'stream';
+
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+interface MinioObjectStorageInput {
+  bucket: string;
+  access_key: string;
+  access_secret: string;
+  useSSL?: boolean;
+  endPoint: string;
+  port: number;
+}
 
 export default class Minio implements IStorageAdapterV2 {
   private minioClient: MinioClient;
-  private input: any;
+  private input: MinioObjectStorageInput;
-
-  constructor(input: any) {
-    this.input = input;
-  }
-
-  async fileCreate(key: string, file: XcFile): Promise<any> {
-    const fileStream = fs.createReadStream(file.path);
-    return this.fileCreateByStream(key, fileStream, {
-      mimetype: file?.mimetype,
-    });
-  }
 
-  public async fileDelete(_path: string): Promise<any> {
+  constructor(input: unknown) {
-    return Promise.resolve(undefined);
+    this.input = input as MinioObjectStorageInput;
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.minioClient.getObject(this.input.bucket, key, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data) {
-          return reject(data);
-        }
-        return resolve(data);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    // todo:  update in ui(checkbox and number field)
+    const minioOptions = {
-    this.input.port = +this.input.port || 9000;
+      port: +this.input.port || 9000,
-    this.input.useSSL = this.input.useSSL === true;
+      endPoint: this.input.endPoint,
-    this.input.accessKey = this.input.access_key;
+      useSSL: this.input.useSSL === true,
-    this.input.secretKey = this.input.access_secret;
+      accessKey: this.input.access_key,
+      secretKey: this.input.access_secret,
+    };
 
-    this.minioClient = new MinioClient(this.input);
+    this.minioClient = new MinioClient(minioOptions);
   }
 
   public async test(): Promise<boolean> {
     try {
-      const tempFile = generateTempFilePath();
+      const createStream = Readable.from(['Hello from Minio, NocoDB']);
-      const createStream = fs.createWriteStream(tempFile);
+      await this.fileCreateByStream('nc-test-file.txt', createStream);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: '',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
       return true;
     } catch (e) {
       throw e;
     }
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  public async fileRead(key: string): Promise<any> {
-    const uploadParams: any = {
+    const data = await this.minioClient.getObject(this.input.bucket, key);
-      ACL: 'public-read',
+
-    };
     return new Promise((resolve, reject) => {
-      axios
+      const chunks: any[] = [];
-        .get(url, {
+      data.on('data', (chunk) => {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        chunks.push(chunk);
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          const metaData = {
-            // 'Content-Type': file.mimetype
-            // 'X-Amz-Meta-Testing': 1234,
-            // 'run': 5678
-          };
-          // call S3 to retrieve upload file to specified bucket
-          this.minioClient
-            .putObject(this.input?.bucket, key, response.data, metaData)
-            .then(() => {
-              resolve({
-                url: `http${this.input.useSSL ? 's' : ''}://${
-                  this.input.endPoint
-                }:${this.input.port}/${this.input.bucket}/${key}`,
-                data: response.data,
       });
-            })
+
-            .catch(reject);
+      data.on('end', () => {
-        })
+        const buffer = Buffer.concat(chunks);
-        .catch((error) => {
+        resolve(buffer);
-          reject(error);
+      });
+
+      data.on('error', (err) => {
+        reject(err);
       });
     });
   }
 
+  async fileCreate(key: string, file: XcFile): Promise<any> {
+    const fileStream = fs.createReadStream(file.path);
+    return this.fileCreateByStream(key, fileStream, {
+      mimetype: file?.mimetype,
+    });
+  }
+
   async fileCreateByStream(
     key: string,
     stream: Readable,
     options?: {
       mimetype?: string;
+      size?: number;
     },
   ): Promise<any> {
-    return new Promise((resolve, reject) => {
+    try {
-      // uploadParams.Body = fileStream;
+      const streamError = new Promise<void>((_, reject) => {
-      // uploadParams.Key = key;
+        stream.on('error', (err) => {
-      const metaData = {
+          reject(err);
-        'Content-Type': options?.mimetype,
+        });
-        // 'X-Amz-Meta-Testing': 1234,
+      });
-        // 'run': 5678
+
+      const uploadParams = {
+        Key: key,
+        Body: stream,
+        metaData: {
+          ContentType: options?.mimetype,
+          size: options?.size,
+        },
+      };
+
+      const upload = this.upload(uploadParams);
+
+      return await Promise.race([upload, streamError]);
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  async fileCreateByUrl(
+    key: string,
+    url: string,
+    { fetchOptions: { buffer } = { buffer: false } },
+  ): Promise<any> {
+    const response = await axios.get(url, {
+      httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+      httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+      responseType: buffer ? 'arraybuffer' : 'stream',
+    });
+
+    const uploadParams = {
+      ACL: 'public-read',
+      Key: key,
+      Body: response.data,
+      metaData: {
+        ContentType: response.headers['content-type'],
+      },
     };
-      // call S3 to retrieve upload file to specified bucket
+
-      this.minioClient
+    const responseUrl = this.upload(uploadParams);
-        .putObject(this.input?.bucket, key, stream, metaData)
+
-        .then(() => {
+    return {
-          resolve(
+      url: responseUrl,
-            `http${this.input.useSSL ? 's' : ''}://${this.input.endPoint}:${
+      data: response.data,
+    };
+  }
+
+  private async upload(uploadParams: {
+    Key: string;
+    Body: Readable;
+    metaData: { [key: string]: string | number };
+  }): Promise<any> {
+    try {
+      this.minioClient.putObject(
+        this.input.bucket,
+        uploadParams.Key,
+        uploadParams.Body,
+        uploadParams.metaData as any,
+      );
+
+      if (this.input.useSSL && this.input.port === 443) {
+        return `https://${this.input.endPoint}/${uploadParams.Key}`;
+      } else if (!this.input.useSSL && this.input.port === 80) {
+        return `http://${this.input.endPoint}/${uploadParams.Key}`;
+      } else {
+        return `http${this.input.useSSL ? 's' : ''}://${this.input.endPoint}:${
           this.input.port
-            }/${this.input.bucket}/${key}`,
+        }/${uploadParams.Key}`;
+      }
+    } catch (error) {
+      console.error('Error uploading file', error);
+      throw error;
+    }
+  }
+
+  public async getSignedUrl(
+    key,
+    expiresInSeconds = 7200,
+    pathParameters?: { [key: string]: string },
+  ) {
+    if (
+      key.startsWith(`${this.input.bucket}/nc/uploads`) ||
+      key.startsWith(`${this.input.bucket}/nc/thumbnails`)
+    ) {
+      key = key.replace(`${this.input.bucket}/`, '');
+    }
+
+    return this.minioClient.presignedGetObject(
+      this.input.bucket,
+      key,
+      expiresInSeconds,
+      pathParameters,
     );
-        })
-        .catch(reject);
-    });
   }
 
   // TODO - implement
@@ -147,4 +188,8 @@ export default class Minio implements IStorageAdapterV2 {
   getDirectoryList(_path: string): Promise<string[]> {
     return Promise.resolve(undefined);
   }
+
+  public async fileDelete(_path: string): Promise<any> {
+    return Promise.resolve(undefined);
+  }
 }

packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts

@@ -1,156 +1,42 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import axios from 'axios';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { useAgent } from 'request-filtering-agent';
+
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+interface OvhCloudStorageInput {
-import type { Readable } from 'stream';
+  bucket: string;
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  region: string;
-
+  access_key: string;
-export default class OvhCloud implements IStorageAdapterV2 {
+  access_secret: string;
-  private s3Client: AWS.S3;
+  acl?: string;
-  private input: any;
+}
-
-  constructor(input: any) {
-    this.input = input;
-  }
-
-  async fileCreate(key: string, file: XcFile): Promise<any> {
-    const fileStream = fs.createReadStream(file.path);
-
-    return this.fileCreateByStream(key, fileStream, {
-      mimetype: file?.mimetype,
-    });
-  }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+export default class OvhCloud extends GenericS3 implements IStorageAdapterV2 {
-    const uploadParams: any = {
+  protected input: OvhCloudStorageInput;
-      ACL: 'public-read',
-    };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
 
-          // call S3 to retrieve upload file to specified bucket
+  constructor(input: unknown) {
-          this.s3Client.upload(uploadParams, (err1, data) => {
+    super(input as OvhCloudStorageInput);
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
-            }
-            if (data) {
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
   }
 
-  async fileCreateByStream(
+  protected get defaultParams() {
-    key: string,
+    return {
-    stream: Readable,
+      Bucket: this.input.bucket,
-    options?: {
+      ACL: this.input?.acl || 'public-read',
-      mimetype?: string;
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      //  ContentType: file.mimetype,
     };
-    return new Promise((resolve, reject) => {
-      // Configure the file stream and obtain the upload parameters
-
-      uploadParams.Body = stream;
-      uploadParams.Key = key;
-      uploadParams.ContentType =
-        options?.mimetype || 'application/octet-stream';
-
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    const s3Options: any = {
+    const s3Options: S3ClientConfig = {
-      params: { Bucket: this.input.bucket },
       region: this.input.region,
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+      // TODO: Need to verify
+      // DOCS s3.<region_in_lowercase>.io.cloud.ovh.net
+      endpoint: `https://s3.${this.input.region}.cloud.ovh.net`,
     };
 
-    s3Options.accessKeyId = this.input.access_key;
+    this.s3Client = new S3Client(s3Options);
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(
-      `s3.${this.input.region}.cloud.ovh.net`,
-    );
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 }

packages/nocodb/src/plugins/ovhCloud/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: OvhCloud,
   title: 'OvhCloud Object Storage',
-  version: '0.0.1',
+  version: '0.0.2',
   logo: 'plugins/ovhCloud.png',
   tags: 'Storage',
   description:
@@ -41,6 +41,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/r2/R2.ts

@@ -0,0 +1,42 @@
+import { S3 as S3Client } from '@aws-sdk/client-s3';
+
+import type { S3ClientConfigType } from '@aws-sdk/client-s3';
+import type { IStorageAdapterV2 } from 'nc-plugin';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
+
+interface R2ObjectStorageInput {
+  bucket: string;
+  access_key: string;
+  access_secret: string;
+  hostname: string;
+  region: string;
+}
+
+export default class R2 extends GenericS3 implements IStorageAdapterV2 {
+  protected input: R2ObjectStorageInput;
+
+  constructor(input: unknown) {
+    super(input as R2ObjectStorageInput);
+  }
+
+  protected get defaultParams() {
+    return {
+      Bucket: this.input.bucket,
+      // R2 does not support ACL
+      ACL: 'private',
+    };
+  }
+
+  public async init(): Promise<any> {
+    const s3Options: S3ClientConfigType = {
+      region: 'auto',
+      endpoint: this.input.hostname,
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+    };
+
+    this.s3Client = new S3Client(s3Options);
+  }
+}

packages/nocodb/src/plugins/r2/R2Plugin.ts

@@ -0,0 +1,18 @@
+import { XcStoragePlugin } from 'nc-plugin';
+import R2 from './R2';
+import type { IStorageAdapterV2 } from 'nc-plugin';
+
+class R2Plugin extends XcStoragePlugin {
+  private static storageAdapter: R2;
+
+  public getAdapter(): IStorageAdapterV2 {
+    return R2Plugin.storageAdapter;
+  }
+
+  public async init(config: any): Promise<any> {
+    R2Plugin.storageAdapter = new R2(config);
+    await R2Plugin.storageAdapter.init();
+  }
+}
+
+export default R2Plugin;

packages/nocodb/src/plugins/r2/index.ts

@@ -0,0 +1,68 @@
+import { XcActionType, XcType } from 'nocodb-sdk';
+import R2Plugin from './R2Plugin';
+import type { XcPluginConfig } from 'nc-plugin';
+
+const config: XcPluginConfig = {
+  builder: R2Plugin,
+  title: 'Cloudflare R2 Storage',
+  version: '0.0.1',
+  logo: 'plugins/r2.png',
+  description:
+    'Cloudflare R2 is an S3-compatible, zero egress-fee, globally distributed object storage.',
+  tags: 'Storage',
+  inputs: {
+    title: 'Configure Cloudflare R2 Storage',
+    items: [
+      {
+        key: 'bucket',
+        label: 'Bucket Name',
+        placeholder: 'Bucket Name',
+        type: XcType.SingleLineText,
+        required: true,
+      },
+      {
+        key: 'hostname',
+        label: 'Host Name',
+        placeholder: 'e.g.: *****.r2.cloudflarestorage.com',
+        type: XcType.SingleLineText,
+        required: true,
+      },
+      {
+        key: 'access_key',
+        label: 'Access Key',
+        placeholder: 'Access Key',
+        type: XcType.SingleLineText,
+        required: true,
+      },
+      {
+        key: 'access_secret',
+        label: 'Access Secret',
+        placeholder: 'Access Secret',
+        type: XcType.Password,
+        required: true,
+      },
+    ],
+    actions: [
+      {
+        label: 'Test',
+        placeholder: 'Test',
+        key: 'test',
+        actionType: XcActionType.TEST,
+        type: XcType.Button,
+      },
+      {
+        label: 'Save',
+        placeholder: 'Save',
+        key: 'save',
+        actionType: XcActionType.SUBMIT,
+        type: XcType.Button,
+      },
+    ],
+    msgOnInstall:
+      'Successfully installed and attachment will be stored in Cloudflare R2 Storage',
+    msgOnUninstall: '',
+  },
+  category: 'Storage',
+};
+
+export default config;

packages/nocodb/src/plugins/s3/S3.ts

@@ -1,163 +1,33 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
-import { GetObjectCommand, S3 as S3Client } from '@aws-sdk/client-s3';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import { Upload } from '@aws-sdk/lib-storage';
-import axios from 'axios';
-import { useAgent } from 'request-filtering-agent';
 import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import type { Readable } from 'stream';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+
+interface S3Input {
+  bucket: string;
+  region: string;
+  access_key: string;
+  access_secret: string;
+  endpoint?: string;
+  acl?: string;
+}
 
-export default class S3 implements IStorageAdapterV2 {
+export default class S3 extends GenericS3 implements IStorageAdapterV2 {
-  private s3Client: S3Client;
+  protected input: S3Input;
-  private input: any;
 
   constructor(input: any) {
-    this.input = input;
+    super(input as S3Input);
   }
 
   get defaultParams() {
     return {
-      ACL: 'private',
+      ACL: this.input.acl || 'private',
       Bucket: this.input.bucket,
     };
   }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
-    // create file stream
-    const fileStream = fs.createReadStream(file.path);
-    // upload using stream
-    return this.fileCreateByStream(key, fileStream, {
-      mimetype: file?.mimetype,
-    });
-  }
-
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
-    const uploadParams: any = {
-      ...this.defaultParams,
-    };
-
-    try {
-      const response = await axios.get(url, {
-        httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-        httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-        // TODO add an extra options argument to pass responseType & use stream for non-image files
-        responseType: 'arraybuffer',
-      });
-
-      uploadParams.Body = response.data;
-      uploadParams.Key = key;
-      uploadParams.ContentType = response.headers['content-type'];
-
-      const data = await this.upload(uploadParams);
-      return {
-        url: data,
-        data: response.data,
-      };
-    } catch (error) {
-      throw error;
-    }
-  }
-
-  fileCreateByStream(
-    key: string,
-    stream: Readable,
-    options?: {
-      mimetype?: string;
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ...this.defaultParams,
-    };
-    return new Promise((resolve, reject) => {
-      stream.on('error', (err) => {
-        console.log('File Error', err);
-        reject(err);
-      });
-
-      uploadParams.Body = stream;
-      uploadParams.Key = key;
-      uploadParams.ContentType =
-        options?.mimetype || 'application/octet-stream';
-
-      // call S3 to upload file to specified bucket
-      this.upload(uploadParams)
-        .then((data) => {
-          resolve(data);
-        })
-        .catch((err) => {
-          reject(err);
-        });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    const command = new GetObjectCommand({
-      Key: key,
-      Bucket: this.input.bucket,
-    });
-
-    const { Body } = await this.s3Client.send(command);
-
-    const fileStream = Body as Readable;
-
-    return new Promise((resolve, reject) => {
-      const chunks: any[] = [];
-      fileStream.on('data', (chunk) => {
-        chunks.push(chunk);
-      });
-
-      fileStream.on('end', () => {
-        const buffer = Buffer.concat(chunks);
-        resolve(buffer);
-      });
-
-      fileStream.on('error', (err) => {
-        reject(err);
-      });
-    });
-  }
-
-  public async getSignedUrl(
-    key,
-    expiresInSeconds = 7200,
-    pathParameters?: { [key: string]: string },
-  ) {
-    const command = new GetObjectCommand({
-      Key: key,
-      Bucket: this.input.bucket,
-      ...pathParameters,
-    });
-    return getSignedUrl(this.s3Client, command, {
-      expiresIn: expiresInSeconds,
-    });
-  }
-
   public async init(): Promise<any> {
-    // const s3Options: any = {
-    //   params: {Bucket: process.env.NC_S3_BUCKET},
-    //   region: process.env.NC_S3_REGION
-    // };
-    //
-    // s3Options.accessKeyId = process.env.NC_S3_KEY;
-    // s3Options.secretAccessKey = process.env.NC_S3_SECRET;
-
     const s3Options: S3ClientConfig = {
       region: this.input.region,
       credentials: {
@@ -173,27 +43,8 @@ export default class S3 implements IStorageAdapterV2 {
     this.s3Client = new S3Client(s3Options);
   }
 
-  public async test(): Promise<boolean> {
+  protected async upload(uploadParams): Promise<any> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
-  }
-
-  private async upload(uploadParams): Promise<any> {
     try {
-      // call S3 to retrieve upload file to specified bucket
       const upload = new Upload({
         client: this.s3Client,
         params: { ...this.defaultParams, ...uploadParams },

packages/nocodb/src/plugins/s3/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: S3Plugin,
   title: 'S3',
-  version: '0.0.2',
+  version: '0.0.3',
   logo: 'plugins/s3.png',
   description:
     'Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.',
@@ -47,6 +47,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to private',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts

@@ -1,149 +1,44 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+
-import axios from 'axios';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import { useAgent } from 'request-filtering-agent';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+
-import type { Readable } from 'stream';
+interface ScalewayObjectStorageInput {
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  bucket: string;
-
+  region: string;
-export default class ScalewayObjectStorage implements IStorageAdapterV2 {
+  access_key: string;
-  private s3Client: AWS.S3;
+  access_secret: string;
-  private input: any;
+  acl?: string;
-
+}
-  constructor(input: any) {
-    this.input = input;
-  }
 
-  public async fileRead(key: string): Promise<any> {
+export default class ScalewayObjectStorage
-    return new Promise((resolve, reject) => {
+  extends GenericS3
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
+  implements IStorageAdapterV2
-        if (err) {
+{
-          return reject(err);
+  protected input: ScalewayObjectStorageInput;
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
-  }
 
-  public async test(): Promise<boolean> {
+  constructor(input: unknown) {
-    try {
+    super(input as ScalewayObjectStorageInput);
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 
-  public async fileDelete(_path: string): Promise<any> {
+  protected get defaultParams() {
-    return Promise.resolve(undefined);
+    return {
+      Bucket: this.input.bucket,
+      ACL: this.input?.acl || 'public-read',
+    };
   }
 
   public async init(): Promise<any> {
-    const s3Options: any = {
+    const s3Options: S3ClientConfig = {
-      params: { Bucket: this.input.bucket },
       region: this.input.region,
-    };
+      credentials: {
-
+        accessKeyId: this.input.access_key,
-    s3Options.accessKeyId = this.input.access_key;
+        secretAccessKey: this.input.access_secret,
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(`s3.${this.input.region}.scw.cloud`);
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  async fileCreate(key: string, file: XcFile): Promise<any> {
-    const fileStream = fs.createReadStream(file.path);
-
-    return this.fileCreateByStream(key, fileStream, {
-      mimetype: file?.mimetype,
-    });
-  }
-
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-    };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          // call S3 to retrieve upload file to specified bucket
-          this.s3Client.upload(uploadParams, (err1, data) => {
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
-            }
-            if (data) {
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
-  }
-
-  async fileCreateByStream(
-    key: string,
-    stream: Readable,
-    options?: {
-      mimetype?: string;
       },
-  ): Promise<any> {
+      endpoint: `https://s3.${this.input.region}.scw.cloud`,
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Body: stream,
-      Key: key,
-      ContentType: options?.mimetype || 'application/octet-stream',
     };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
 
-  // TODO - implement
+    this.s3Client = new S3Client(s3Options);
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
   }
 }

packages/nocodb/src/plugins/scaleway/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: ScalewayObjectStoragePlugin,
   title: 'Scaleway Object Storage',
-  version: '0.0.1',
+  version: '0.0.2',
   logo: 'plugins/scaleway.png',
   tags: 'Storage',
   description:
@@ -41,6 +41,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/ses/SES.ts

@@ -1,5 +1,6 @@
 import nodemailer from 'nodemailer';
-import AWS from 'aws-sdk';
+
+import { SendRawEmailCommand, SES as SESClient } from '@aws-sdk/client-ses';
 import type { IEmailAdapter } from 'nc-plugin';
 import type Mail from 'nodemailer/lib/mailer';
 import type { XcEmail } from '~/interface/IEmailAdapter';
@@ -13,14 +14,20 @@ export default class SES implements IEmailAdapter {
   }
 
   public async init(): Promise<any> {
-    const sesOptions: any = {
+    const ses = new SESClient({
+      apiVersion: '2006-03-01',
+      region: this.input.region,
+      credentials: {
         accessKeyId: this.input.access_key,
         secretAccessKey: this.input.access_secret,
-      region: this.input.region,
+      },
-    };
+    });
 
     this.transporter = nodemailer.createTransport({
-      SES: new AWS.SES(sesOptions),
+      SES: {
+        ses,
+        aws: { SendRawEmailCommand },
+      },
     });
   }
 

packages/nocodb/src/plugins/spaces/Spaces.ts

@@ -1,159 +1,41 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import axios from 'axios';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { useAgent } from 'request-filtering-agent';
+
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+interface SpacesObjectStorageInput {
-import type { Readable } from 'stream';
+  bucket: string;
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  region: string;
-
+  access_key: string;
-export default class Spaces implements IStorageAdapterV2 {
+  access_secret: string;
-  private s3Client: AWS.S3;
+  acl?: string;
-  private input: any;
+}
-
-  constructor(input: any) {
-    this.input = input;
-  }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
+export default class Spaces extends GenericS3 implements IStorageAdapterV2 {
-    const fileStream = fs.createReadStream(file.path);
+  protected input: SpacesObjectStorageInput;
 
-    return this.fileCreateByStream(key, fileStream, {
+  constructor(input: unknown) {
-      mimetype: file?.mimetype,
+    super(input as SpacesObjectStorageInput);
-    });
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  protected get defaultParams() {
-    const uploadParams: any = {
+    return {
-      ACL: 'public-read',
+      Bucket: this.input.bucket,
+      ACL: this.input?.acl || 'public-read',
     };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          // call S3 to retrieve upload file to specified bucket
-          this.s3Client.upload(uploadParams, (err1, data) => {
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
-            }
-            if (data) {
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
-  }
-
-  async fileCreateByStream(
-    key: string,
-    stream: Readable,
-    options?: {
-      mimetype?: string;
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Body: stream,
-      Key: key,
-      ContentType: options?.mimetype || 'application/octet-stream',
-    };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    // const s3Options: any = {
+    const s3Options: S3ClientConfig = {
-    //   params: {Bucket: process.env.NC_S3_BUCKET},
+      region: 'us-east-1',
-    //   region: process.env.NC_S3_REGION
+      forcePathStyle: false,
-    // };
+      credentials: {
-    //
+        accessKeyId: this.input.access_key,
-    // s3Options.accessKeyId = process.env.NC_S3_KEY;
+        secretAccessKey: this.input.access_secret,
-    // s3Options.secretAccessKey = process.env.NC_S3_SECRET;
+      },
-
+      endpoint: `https://${this.input.region || 'nyc3'}.digitaloceanspaces.com`,
-    const s3Options: any = {
-      params: { Bucket: this.input.bucket },
-      region: this.input.region,
     };
 
-    s3Options.accessKeyId = this.input.access_key;
+    this.s3Client = new S3Client(s3Options);
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(
-      `${this.input.region || 'nyc3'}.digitaloceanspaces.com`,
-    );
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 }

packages/nocodb/src/plugins/spaces/index.ts

@@ -43,6 +43,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/storage/Local.ts

@@ -24,12 +24,16 @@ export default class Local implements IStorageAdapterV2 {
     }
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  async fileCreateByUrl(
+    key: string,
+    url: string,
+    { fetchOptions: { buffer } = { buffer: false } },
+  ): Promise<any> {
     const destPath = validateAndNormaliseLocalPath(key);
     return new Promise((resolve, reject) => {
       axios
         .get(url, {
-          responseType: 'arraybuffer',
+          responseType: buffer ? 'arraybuffer' : 'stream',
           headers: {
             accept:
               'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',

packages/nocodb/src/plugins/upcloud/UpoCloud.ts

@@ -1,149 +1,45 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import axios from 'axios';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { useAgent } from 'request-filtering-agent';
+
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+interface UpoCloudStorgeInput {
-import type { Readable } from 'stream';
+  bucket: string;
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  region: string;
-
+  access_key: string;
-export default class UpoCloud implements IStorageAdapterV2 {
+  access_secret: string;
-  private s3Client: AWS.S3;
+  endpoint: string;
-  private input: any;
+  acl?: string;
-
+}
-  constructor(input: any) {
-    this.input = input;
-  }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
+export default class UpoCloud extends GenericS3 implements IStorageAdapterV2 {
-    const fileStream = fs.createReadStream(file.path);
+  protected input: UpoCloudStorgeInput;
 
-    return this.fileCreateByStream(key, fileStream, {
+  constructor(input: unknown) {
-      mimetype: file?.mimetype,
+    super(input as UpoCloudStorgeInput);
-    });
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  protected get defaultParams() {
-    const uploadParams: any = {
+    return {
-      ACL: 'public-read',
+      Bucket: this.input.bucket,
+      ACL: this.input?.acl || 'public-read',
     };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          // call S3 to retrieve upload file to specified bucket
-          this.s3Client.upload(uploadParams, (err1, data) => {
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
-            }
-            if (data) {
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
-  }
-
-  async fileCreateByStream(
-    key: string,
-    stream: Readable,
-    options?: {
-      mimetype?: string;
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Key: key,
-      Body: stream,
-      ContentType: options?.mimetype || 'application/octet-stream',
-    };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    const s3Options: any = {
+    const updatedEndpoint = this.input.endpoint.startsWith('https://')
-      params: { Bucket: this.input.bucket },
+      ? this.input.endpoint
+      : `https://${this.input.endpoint}`;
+
+    const s3Options: S3ClientConfig = {
       region: this.input.region,
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+      endpoint: updatedEndpoint,
     };
 
-    s3Options.accessKeyId = this.input.access_key;
+    this.s3Client = new S3Client(s3Options);
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(this.input.endpoint);
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 }

packages/nocodb/src/plugins/upcloud/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: UpCloudPlugin,
   title: 'UpCloud Object Storage',
-  version: '0.0.1',
+  version: '0.0.2',
   logo: 'plugins/upcloud.png',
   description:
     'The perfect home for your data. Thanks to the S3-compatible programmable interface,\n' +
@@ -42,6 +42,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/plugins/vultr/Vultr.ts

@@ -1,149 +1,41 @@
-import fs from 'fs';
+import { S3 as S3Client } from '@aws-sdk/client-s3';
-import { promisify } from 'util';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
-import AWS from 'aws-sdk';
+import type { IStorageAdapterV2 } from 'nc-plugin';
-import axios from 'axios';
+import GenericS3 from '~/plugins/GenericS3/GenericS3';
-import { useAgent } from 'request-filtering-agent';
+
-import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
+interface VultrObjectStorageInput {
-import type { Readable } from 'stream';
+  bucket: string;
-import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
+  region: string;
-
+  access_key: string;
-export default class Vultr implements IStorageAdapterV2 {
+  hostname: string;
-  private s3Client: AWS.S3;
+  access_secret: string;
-  private input: any;
+  acl?: string;
-
+}
-  constructor(input: any) {
-    this.input = input;
-  }
 
-  async fileCreate(key: string, file: XcFile): Promise<any> {
+export default class Vultr extends GenericS3 implements IStorageAdapterV2 {
-    const fileStream = fs.createReadStream(file.path);
+  protected input: VultrObjectStorageInput;
 
-    return this.fileCreateByStream(key, fileStream, {
+  constructor(input: unknown) {
-      mimetype: file?.mimetype,
+    super(input as VultrObjectStorageInput);
-    });
   }
 
-  async fileCreateByUrl(key: string, url: string): Promise<any> {
+  protected get defaultParams() {
-    const uploadParams: any = {
+    return {
-      ACL: 'public-read',
+      Bucket: this.input.bucket,
+      ACL: this.input?.acl || 'public-read',
     };
-    return new Promise((resolve, reject) => {
-      axios
-        .get(url, {
-          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
-          // TODO - use stream instead of buffer
-          responseType: 'arraybuffer',
-        })
-        .then((response) => {
-          uploadParams.Body = response.data;
-          uploadParams.Key = key;
-          uploadParams.ContentType = response.headers['content-type'];
-
-          // call S3 to retrieve upload file to specified bucket
-          this.s3Client.upload(uploadParams, (err1, data) => {
-            if (err1) {
-              console.log('Error', err1);
-              reject(err1);
-            }
-            if (data) {
-              resolve({
-                url: data.Location,
-                data: response.data,
-              });
-            }
-          });
-        })
-        .catch((error) => {
-          reject(error);
-        });
-    });
-  }
-
-  async fileCreateByStream(
-    key: string,
-    stream: Readable,
-    options?: {
-      mimetype: string;
-    },
-  ): Promise<void> {
-    const uploadParams: any = {
-      ACL: 'public-read',
-      Body: stream,
-      Key: key,
-      ContentType: options?.mimetype || 'application/octet-stream',
-    };
-    return new Promise((resolve, reject) => {
-      // call S3 to retrieve upload file to specified bucket
-      this.s3Client.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.log('Error', err);
-          reject(err);
-        }
-        if (data) {
-          resolve(data.Location);
-        }
-      });
-    });
-  }
-
-  // TODO - implement
-  fileReadByStream(_key: string): Promise<Readable> {
-    return Promise.resolve(undefined);
-  }
-
-  // TODO - implement
-  getDirectoryList(_path: string): Promise<string[]> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileDelete(_path: string): Promise<any> {
-    return Promise.resolve(undefined);
-  }
-
-  public async fileRead(key: string): Promise<any> {
-    return new Promise((resolve, reject) => {
-      this.s3Client.getObject({ Key: key } as any, (err, data) => {
-        if (err) {
-          return reject(err);
-        }
-        if (!data?.Body) {
-          return reject(data);
-        }
-        return resolve(data.Body);
-      });
-    });
   }
 
   public async init(): Promise<any> {
-    const s3Options: any = {
+    const s3Options: S3ClientConfig = {
-      params: { Bucket: this.input.bucket },
       region: this.input.region,
+      credentials: {
+        accessKeyId: this.input.access_key,
+        secretAccessKey: this.input.access_secret,
+      },
+      endpoint: this.input.hostname,
     };
 
-    s3Options.accessKeyId = this.input.access_key;
+    this.s3Client = new S3Client(s3Options);
-    s3Options.secretAccessKey = this.input.access_secret;
-
-    s3Options.endpoint = new AWS.Endpoint(this.input.hostname);
-
-    this.s3Client = new AWS.S3(s3Options);
-  }
-
-  public async test(): Promise<boolean> {
-    try {
-      const tempFile = generateTempFilePath();
-      const createStream = fs.createWriteStream(tempFile);
-      await waitForStreamClose(createStream);
-      await this.fileCreate('nc-test-file.txt', {
-        path: tempFile,
-        mimetype: 'text/plain',
-        originalname: 'temp.txt',
-        size: '',
-      });
-      await promisify(fs.unlink)(tempFile);
-      return true;
-    } catch (e) {
-      throw e;
-    }
   }
 }

packages/nocodb/src/plugins/vultr/index.ts

@@ -5,7 +5,7 @@ import type { XcPluginConfig } from 'nc-plugin';
 const config: XcPluginConfig = {
   builder: VultrPlugin,
   title: 'Vultr Object Storage',
-  version: '0.0.2',
+  version: '0.0.3',
   logo: 'plugins/vultr.png',
   description:
     'Using Vultr Object Storage can give flexibility and cloud storage that allows applications greater flexibility and access worldwide.',
@@ -41,6 +41,13 @@ const config: XcPluginConfig = {
         type: XcType.Password,
         required: true,
       },
+      {
+        key: 'acl',
+        label: 'Access Control Lists (ACL)',
+        placeholder: 'Default set to public-read',
+        type: XcType.SingleLineText,
+        required: false,
+      },
     ],
     actions: [
       {

packages/nocodb/src/services/attachments.service.ts

@@ -208,6 +208,12 @@ export class AttachmentsService {
             await storageAdapter.fileCreateByUrl(
               slash(path.join(destPath, fileName)),
               finalUrl,
+              {
+                fetchOptions: {
+                  // The sharp requires image to be passed as buffer.);
+                  buffer: mimeType.includes('image'),
+                },
+              },
             );
 
           const tempMetadata: {