Merge pull request #6711 from nocodb/nc-fix/bucket-attachments

fix: upload via url to buckets

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -1,4 +1,5 @@
 import { promisify } from 'util';
+import moment from 'moment';
 import { UITypes } from 'nocodb-sdk';
 import Airtable from 'airtable';
 import jsonfile from 'jsonfile';
@@ -1544,8 +1545,11 @@ export class AtImportProcessor {
                     ?.map((a) => a.filename?.split('?')?.[0])
                     .join(', ')}`,
                 );
+                const path = `${moment().format('YYYY/MM/DD')}/${hash(
+                  syncDB.user.id,
+                )}`;
                 tempArr = await this.attachmentsService.uploadViaURL({
-                  path: `noco/${sDB.baseName}/${table.title}/${key}`,
+                  path,
                   urls: value?.map((attachment) => ({
                     fileName: attachment.filename?.split('?')?.[0],
                     url: attachment.url,

packages/nocodb/src/plugins/backblaze/Backblaze.ts

@@ -54,6 +54,7 @@ export default class Backblaze implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/gcs/Gcs.ts

@@ -110,6 +110,7 @@ export default class Gcs implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           this.storageClient

packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts

@@ -53,6 +53,7 @@ export default class LinodeObjectStorage implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/mino/Minio.ts

@@ -100,6 +100,7 @@ export default class Minio implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts

@@ -53,6 +53,7 @@ export default class OvhCloud implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/s3/S3.ts

@@ -60,6 +60,7 @@ export default class S3 implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts

@@ -103,6 +103,7 @@ export default class ScalewayObjectStorage implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/spaces/Spaces.ts

@@ -53,6 +53,7 @@ export default class Spaces implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/upcloud/UpoCloud.ts

@@ -53,6 +53,7 @@ export default class UpoCloud implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/plugins/vultr/Vultr.ts

@@ -53,6 +53,7 @@ export default class Vultr implements IStorageAdapterV2 {
         .get(url, {
           httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
           httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          responseType: 'arraybuffer',
         })
         .then((response) => {
           uploadParams.Body = response.data;

packages/nocodb/src/services/attachments.service.ts

@@ -111,7 +111,9 @@ export class AttachmentsService {
       param.urls?.map?.(async (urlMeta) => {
         const { url, fileName: _fileName } = urlMeta;
 
-        const fileName = `${nanoid(18)}${_fileName || url.split('/').pop()}`;
+        const fileName = `${nanoid(18)}${path.extname(
+          _fileName || url.split('/').pop(),
+        )}`;
 
         const attachmentUrl = await (storageAdapter as any).fileCreateByUrl(
           slash(path.join(destPath, fileName)),
@@ -130,7 +132,7 @@ export class AttachmentsService {
         return {
           ...(attachmentUrl ? { url: attachmentUrl } : {}),
           ...(attachmentPath ? { path: attachmentPath } : {}),
-          title: fileName,
+          title: _fileName,
           mimetype: urlMeta.mimetype,
           size: urlMeta.size,
           icon: mimeIcons[path.extname(fileName).slice(1)] || undefined,