feat: attachment apis (WIP)

Signed-off-by: Pranav C <pranavxc@gmail.com>

packages/nocodb-nest/package-lock.json

@@ -15,7 +15,7 @@
         "@nestjs/core": "^9.0.0",
         "@nestjs/jwt": "^10.0.3",
         "@nestjs/mapped-types": "*",
-        "@nestjs/platform-express": "^9.0.0",
+        "@nestjs/platform-express": "^9.4.0",
         "@sentry/node": "^6.3.5",
         "airtable": "^0.11.3",
         "ajv": "^8.12.0",
@@ -68,7 +68,7 @@
         "mkdirp": "^2.1.3",
         "morgan": "^1.10.0",
         "mssql": "^6.2.0",
-        "multer": "^1.4.2",
+        "multer": "^1.4.4",
         "mysql2": "^3.2.0",
         "nanoid": "^3.1.20",
         "nc-help": "^0.2.87",
@@ -2486,9 +2486,9 @@
       }
     },
     "node_modules/@nestjs/platform-express": {
-      "version": "9.3.12",
+      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.3.12.tgz",
+      "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.0.tgz",
-      "integrity": "sha512-iQToH9rnZHmm3a2YDKLEN7weU2qC/EVOBnuwTf1lIkqB48yLxlykSJu3KmgtlUUNDt2/HY527QIo+GZSZfCLyg==",
+      "integrity": "sha512-PpnfghpNq7mwG43z3+pacHulsabUCBMla4nUikntXT525ORpZSDvh/nLi1HLfE4w5+FcINc8/RBOyYTeRVmiRQ==",
       "dependencies": {
         "body-parser": "1.20.2",
         "cors": "2.8.5",
@@ -18597,9 +18597,9 @@
       "requires": {}
     },
     "@nestjs/platform-express": {
-      "version": "9.3.12",
+      "version": "9.4.0",
-      "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.3.12.tgz",
+      "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.0.tgz",
-      "integrity": "sha512-iQToH9rnZHmm3a2YDKLEN7weU2qC/EVOBnuwTf1lIkqB48yLxlykSJu3KmgtlUUNDt2/HY527QIo+GZSZfCLyg==",
+      "integrity": "sha512-PpnfghpNq7mwG43z3+pacHulsabUCBMla4nUikntXT525ORpZSDvh/nLi1HLfE4w5+FcINc8/RBOyYTeRVmiRQ==",
       "requires": {
         "body-parser": "1.20.2",
         "cors": "2.8.5",

packages/nocodb-nest/package.json

@@ -26,7 +26,7 @@
     "@nestjs/core": "^9.0.0",
     "@nestjs/jwt": "^10.0.3",
     "@nestjs/mapped-types": "*",
-    "@nestjs/platform-express": "^9.0.0",
+    "@nestjs/platform-express": "^9.4.0",
     "@sentry/node": "^6.3.5",
     "airtable": "^0.11.3",
     "ajv": "^8.12.0",
@@ -79,7 +79,7 @@
     "mkdirp": "^2.1.3",
     "morgan": "^1.10.0",
     "mssql": "^6.2.0",
-    "multer": "^1.4.2",
+    "multer": "^1.4.4",
     "mysql2": "^3.2.0",
     "nanoid": "^3.1.20",
     "nc-help": "^0.2.87",

packages/nocodb-nest/src/app.module.ts

@@ -29,9 +29,10 @@ import { ProjectUsersModule } from './modules/project-users/project-users.module
 import { ModelVisibilitiesModule } from './modules/model-visibilities/model-visibilities.module';
 import { HookFiltersModule } from './modules/hook-filters/hook-filters.module';
 import { ApiTokensModule } from './modules/api-tokens/api-tokens.module';
+import { AttachmentsModule } from './modules/attachments/attachments.module';
 
 @Module({
-  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule],
+  imports: [AuthModule, UsersModule, UtilsModule, ProjectsModule, TablesModule, ViewsModule, FiltersModule, SortsModule, ColumnsModule, ViewColumnsModule, BasesModule, HooksModule, SharedBasesModule, FormsModule, GridsModule, KanbansModule, GalleriesModule, FormColumnsModule, GridColumnsModule, MapsModule, ProjectUsersModule, ModelVisibilitiesModule, HookFiltersModule, ApiTokensModule, AttachmentsModule],
   controllers: [],
   providers: [Connection, MetaService, JwtStrategy, ExtractProjectIdMiddleware],
   exports: [Connection, MetaService],

packages/nocodb-nest/src/modules/attachments/attachments.controller.spec.ts

@@ -0,0 +1,20 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { AttachmentsController } from './attachments.controller';
+import { AttachmentsService } from './attachments.service';
+
+describe('AttachmentsController', () => {
+  let controller: AttachmentsController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [AttachmentsController],
+      providers: [AttachmentsService],
+    }).compile();
+
+    controller = module.get<AttachmentsController>(AttachmentsController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/attachments/attachments.controller.ts

@@ -0,0 +1,123 @@
+import { Controller } from '@nestjs/common';
+import { AttachmentsService } from './attachments.service';
+
+@Controller('attachments')
+export class AttachmentsController {
+  constructor(private readonly attachmentsService: AttachmentsService) {}
+
+
+  const isUploadAllowedMw = async (req: Request, _res: Response, next: any) => {
+    if (!req['user']?.id) {
+      if (!req['user']?.isPublicBase) {
+        NcError.unauthorized('Unauthorized');
+      }
+    }
+
+    try {
+      // check user is super admin or creator
+      if (
+        req['user'].roles?.includes(OrgUserRoles.SUPER_ADMIN) ||
+        req['user'].roles?.includes(OrgUserRoles.CREATOR) ||
+        req['user'].roles?.includes(ProjectRoles.EDITOR) ||
+        // if viewer then check at-least one project have editor or higher role
+        // todo: cache
+        !!(await Noco.ncMeta
+          .knex(MetaTable.PROJECT_USERS)
+          .where(function () {
+            this.where('roles', ProjectRoles.OWNER);
+            this.orWhere('roles', ProjectRoles.CREATOR);
+            this.orWhere('roles', ProjectRoles.EDITOR);
+          })
+          .andWhere('fk_user_id', req['user'].id)
+          .first())
+      )
+        return next();
+    } catch {}
+    NcError.badRequest('Upload not allowed');
+  };
+
+  export async function upload(req: Request, res: Response) {
+    const attachments = await attachmentService.upload({
+      files: (req as any).files,
+      path: req.query?.path as string,
+    });
+
+    res.json(attachments);
+  }
+
+  export async function uploadViaURL(req: Request, res: Response) {
+    const attachments = await attachmentService.uploadViaURL({
+      urls: req.body,
+      path: req.query?.path as string,
+    });
+
+    res.json(attachments);
+  }
+
+  export async function fileRead(req, res) {
+    try {
+      const { img, type } = await attachmentService.fileRead({
+        path: path.join('nc', 'uploads', req.params?.[0]),
+      });
+
+      res.writeHead(200, { 'Content-Type': type });
+      res.end(img, 'binary');
+    } catch (e) {
+      console.log(e);
+      res.status(404).send('Not found');
+    }
+  }
+
+  const router = Router({ mergeParams: true });
+
+  router.get(
+/^\/dl\/([^/]+)\/([^/]+)\/(.+)$/,
+  getCacheMiddleware(),
+  async (req, res) => {
+  try {
+  const { img, type } = await attachmentService.fileRead({
+  path: path.join(
+    'nc',
+    req.params[0],
+    req.params[1],
+    'uploads',
+    ...req.params[2].split('/')
+  ),
+});
+
+res.writeHead(200, { 'Content-Type': type });
+res.end(img, 'binary');
+} catch (e) {
+  res.status(404).send('Not found');
+}
+}
+);
+
+router.post(
+  '/api/v1/db/storage/upload',
+  multer({
+    storage: multer.diskStorage({}),
+    limits: {
+      fieldSize: NC_ATTACHMENT_FIELD_SIZE,
+    },
+  }).any(),
+  [
+    extractProjectIdAndAuthenticate,
+    catchError(isUploadAllowedMw),
+    catchError(upload),
+  ]
+);
+
+router.post(
+  '/api/v1/db/storage/upload-by-url',
+
+  [
+    extractProjectIdAndAuthenticate,
+    catchError(isUploadAllowedMw),
+    catchError(uploadViaURL),
+  ]
+);
+
+router.get(/^\/download\/(.+)$/, getCacheMiddleware(), catchError(fileRead));
+
+}

packages/nocodb-nest/src/modules/attachments/attachments.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { AttachmentsService } from './attachments.service';
+import { AttachmentsController } from './attachments.controller';
+
+@Module({
+  controllers: [AttachmentsController],
+  providers: [AttachmentsService]
+})
+export class AttachmentsModule {}

packages/nocodb-nest/src/modules/attachments/attachments.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { AttachmentsService } from './attachments.service';
+
+describe('AttachmentsService', () => {
+  let service: AttachmentsService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [AttachmentsService],
+    }).compile();
+
+    service = module.get<AttachmentsService>(AttachmentsService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

packages/nocodb-nest/src/modules/attachments/attachments.service.ts

@@ -0,0 +1,127 @@
+import { Injectable } from '@nestjs/common';
+import { nanoid } from 'nanoid';
+import path from 'path';
+import slash from 'slash';
+import NcPluginMgrv2 from '../../helpers/NcPluginMgrv2';
+import Local from '../../plugins/storage/Local';
+import mimetypes, { mimeIcons } from '../../utils/mimeTypes';
+import { T } from 'nc-help';
+
+@Injectable()
+export class AttachmentsService {
+  async upload(param: {
+    path?: string;
+    // todo: proper type
+    files: unknown[];
+  }) {
+    // TODO: add getAjvValidatorMw
+    const filePath = this.sanitizeUrlPath(
+      param.path?.toString()?.split('/') || [''],
+    );
+    const destPath = path.join('nc', 'uploads', ...filePath);
+
+    const storageAdapter = await NcPluginMgrv2.storageAdapter();
+
+    const attachments = await Promise.all(
+      param.files?.map(async (file: any) => {
+        const fileName = `${nanoid(18)}${path.extname(file.originalname)}`;
+
+        const url = await storageAdapter.fileCreate(
+          slash(path.join(destPath, fileName)),
+          file,
+        );
+
+        let attachmentPath;
+
+        // if `url` is null, then it is local attachment
+        if (!url) {
+          // then store the attachement path only
+          // url will be constructued in `useAttachmentCell`
+          attachmentPath = `download/${filePath.join('/')}/${fileName}`;
+        }
+
+        return {
+          ...(url ? { url } : {}),
+          ...(attachmentPath ? { path: attachmentPath } : {}),
+          title: file.originalname,
+          mimetype: file.mimetype,
+          size: file.size,
+          icon:
+            mimeIcons[path.extname(file.originalname).slice(1)] || undefined,
+        };
+      }),
+    );
+
+    T.emit('evt', { evt_type: 'image:uploaded' });
+
+    return attachments;
+  }
+
+  async uploadViaURL(param: {
+    path?: string;
+    urls: {
+      url: string;
+      fileName: string;
+      mimetype?: string;
+      size?: string | number;
+    }[];
+  }) {
+    // TODO: add getAjvValidatorMw
+    const filePath = this.sanitizeUrlPath(
+      param?.path?.toString()?.split('/') || [''],
+    );
+    const destPath = path.join('nc', 'uploads', ...filePath);
+
+    const storageAdapter = await NcPluginMgrv2.storageAdapter();
+
+    const attachments = await Promise.all(
+      param.urls?.map?.(async (urlMeta) => {
+        const { url, fileName: _fileName } = urlMeta;
+
+        const fileName = `${nanoid(18)}${_fileName || url.split('/').pop()}`;
+
+        const attachmentUrl = await (storageAdapter as any).fileCreateByUrl(
+          slash(path.join(destPath, fileName)),
+          url,
+        );
+
+        let attachmentPath;
+
+        // if `attachmentUrl` is null, then it is local attachment
+        if (!attachmentUrl) {
+          // then store the attachement path only
+          // url will be constructued in `useAttachmentCell`
+          attachmentPath = `download/${filePath.join('/')}/${fileName}`;
+        }
+
+        return {
+          ...(attachmentUrl ? { url: attachmentUrl } : {}),
+          ...(attachmentPath ? { path: attachmentPath } : {}),
+          title: fileName,
+          mimetype: urlMeta.mimetype,
+          size: urlMeta.size,
+          icon: mimeIcons[path.extname(fileName).slice(1)] || undefined,
+        };
+      }),
+    );
+
+    T.emit('evt', { evt_type: 'image:uploaded' });
+
+    return attachments;
+  }
+
+  async fileRead(param: { path: string }) {
+    // get the local storage adapter to display local attachments
+    const storageAdapter = new Local();
+    const type =
+      mimetypes[path.extname(param.path).split('/').pop().slice(1)] ||
+      'text/plain';
+
+    const img = await storageAdapter.fileRead(slash(param.path));
+    return { img, type };
+  }
+
+  sanitizeUrlPath(paths) {
+    return paths.map((url) => url.replace(/[/.?#]+/g, '_'));
+  }
+}