feat: implement filter agent for all request points

packages/nocodb/src/helpers/webhookHelpers.ts

@@ -1,5 +1,7 @@
 import Handlebars from 'handlebars';
 import { v4 as uuidv4 } from 'uuid';
+import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import NcPluginMgrv2 from './NcPluginMgrv2';
 import type { Column, FormView, Hook, Model, View } from '~/models';
 import type { HookLogType } from 'nocodb-sdk';
@@ -182,7 +184,7 @@ export async function handleHttpWebHook(
     user,
     constructWebHookData(hook, model, view, prevData, newData),
   );
-  return require('axios')(req);
+  return axios(req);
 }
 
 export function axiosRequestMake(_apiMeta, _user, data) {
@@ -219,6 +221,8 @@ export function axiosRequestMake(_apiMeta, _user, data) {
     }
   }
   apiMeta.response = {};
+  const url = parseBody(apiMeta.path, data);
+
   const req = {
     params: apiMeta.parameters
       ? apiMeta.parameters.reduce((paramsObj, param) => {
@@ -228,7 +232,7 @@ export function axiosRequestMake(_apiMeta, _user, data) {
           return paramsObj;
         }, {})
       : {},
-    url: parseBody(apiMeta.path, data),
+    url: url,
     method: apiMeta.method,
     data: apiMeta.body,
     headers: apiMeta.headers
@@ -240,6 +244,12 @@ export function axiosRequestMake(_apiMeta, _user, data) {
         }, {})
       : {},
     withCredentials: true,
+    httpAgent: useAgent(url, {
+      stopPortScanningByUrlRedirection: true,
+    }),
+    httpsAgent: useAgent(url, {
+      stopPortScanningByUrlRedirection: true,
+    }),
   };
   return req;
 }

packages/nocodb/src/plugins/backblaze/Backblaze.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -50,7 +51,10 @@ export default class Backblaze implements IStorageAdapterV2 {
 
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/discord/Discord.ts

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IWebhookNotificationAdapter } from 'nc-plugin';
 
 export default class Discord implements IWebhookNotificationAdapter {
@@ -11,6 +12,12 @@ export default class Discord implements IWebhookNotificationAdapter {
       try {
         return await axios.post(webhook_url, {
           content,
+          httpAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
+          httpsAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
         });
       } catch (e) {
         console.log(e);

packages/nocodb/src/plugins/gcs/Gcs.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import { Storage } from '@google-cloud/storage';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import type { StorageOptions } from '@google-cloud/storage';
@@ -106,7 +107,10 @@ export default class Gcs implements IStorageAdapterV2 {
   fileCreateByUrl(destPath: string, url: string): Promise<any> {
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           this.storageClient
             .bucket(this.bucketName)

packages/nocodb/src/plugins/linode/LinodeObjectStorage.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -49,7 +50,10 @@ export default class LinodeObjectStorage implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/mattermost/Mattermost.ts

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IWebhookNotificationAdapter } from 'nc-plugin';
 
 export default class Mattermost implements IWebhookNotificationAdapter {
@@ -11,6 +12,12 @@ export default class Mattermost implements IWebhookNotificationAdapter {
       try {
         return await axios.post(webhook_url, {
           text,
+          httpAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
+          httpsAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
         });
       } catch (e) {
         console.log(e);

packages/nocodb/src/plugins/mino/Minio.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import { Client as MinioClient } from 'minio';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -96,7 +97,10 @@ export default class Minio implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/ovhCloud/OvhCloud.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -49,7 +50,10 @@ export default class OvhCloud implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/s3/S3.ts

@@ -3,6 +3,7 @@ import { promisify } from 'util';
 import { GetObjectCommand, S3 as S3Client } from '@aws-sdk/client-s3';
 import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -56,7 +57,10 @@ export default class S3 implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/scaleway/ScalewayObjectStorage.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -99,7 +100,10 @@ export default class ScalewayObjectStorage implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/slack/Slack.ts

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IWebhookNotificationAdapter } from 'nc-plugin';
 
 export default class Slack implements IWebhookNotificationAdapter {
@@ -11,6 +12,12 @@ export default class Slack implements IWebhookNotificationAdapter {
       try {
         return await axios.post(webhook_url, {
           text,
+          httpAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
+          httpsAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
         });
       } catch (e) {
         console.log(e);

packages/nocodb/src/plugins/spaces/Spaces.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -49,7 +50,10 @@ export default class Spaces implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/storage/Local.ts

@@ -3,6 +3,7 @@ import path from 'path';
 import { promisify } from 'util';
 import mkdirp from 'mkdirp';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { NcError } from '~/helpers/catchError';
@@ -40,6 +41,8 @@ export default class Local implements IStorageAdapterV2 {
               'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
             origin: 'https://www.airtable.com/',
           },
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
         })
         .then(async (response) => {
           await mkdirp(path.dirname(destPath));

packages/nocodb/src/plugins/teams/Teams.ts

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IWebhookNotificationAdapter } from 'nc-plugin';
 
 export default class Teams implements IWebhookNotificationAdapter {
@@ -11,6 +12,12 @@ export default class Teams implements IWebhookNotificationAdapter {
       try {
         return await axios.post(webhook_url, {
           Text,
+          httpAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
+          httpsAgent: useAgent(webhook_url, {
+            stopPortScanningByUrlRedirection: true,
+          }),
         });
       } catch (e) {
         console.log(e);

packages/nocodb/src/plugins/upcloud/UpoCloud.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -49,7 +50,10 @@ export default class UpoCloud implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/plugins/vultr/Vultr.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import { promisify } from 'util';
 import AWS from 'aws-sdk';
 import axios from 'axios';
+import { useAgent } from 'request-filtering-agent';
 import type { IStorageAdapterV2, XcFile } from 'nc-plugin';
 import type { Readable } from 'stream';
 import { generateTempFilePath, waitForStreamClose } from '~/utils/pluginUtils';
@@ -49,7 +50,10 @@ export default class Vultr implements IStorageAdapterV2 {
     };
     return new Promise((resolve, reject) => {
       axios
-        .get(url)
+        .get(url, {
+          httpAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+          httpsAgent: useAgent(url, { stopPortScanningByUrlRedirection: true }),
+        })
         .then((response) => {
           uploadParams.Body = response.data;
           uploadParams.Key = key;

packages/nocodb/src/services/utils.service.ts

@@ -3,6 +3,7 @@ import axios from 'axios';
 import { compareVersions, validate } from 'compare-versions';
 import { ViewTypes } from 'nocodb-sdk';
 import { ConfigService } from '@nestjs/config';
+import { useAgent } from 'request-filtering-agent';
 import type { AppConfig } from '~/interface/config';
 import { NC_ATTACHMENT_FIELD_SIZE } from '~/constants';
 import SqlMgrv2 from '~/db/sql-mgr/v2/SqlMgrv2';
@@ -152,6 +153,12 @@ export class UtilsService {
         : {},
       responseType: apiMeta.responseType || 'json',
       withCredentials: true,
+      httpAgent: useAgent(apiMeta.url, {
+        stopPortScanningByUrlRedirection: true,
+      }),
+      httpsAgent: useAgent(apiMeta.url, {
+        stopPortScanningByUrlRedirection: true,
+      }),
     };
     const data = await axios(_req);
     return data?.data;