Merge branch 'develop' into feat/yyyy-mm

packages/nc-gui/components/cell/attachment/utils.ts

@@ -195,7 +195,6 @@ export const [useProvideAttachmentCell, useAttachmentCell] = useInjectionState(
           },
           {
             files,
-            json: '{}',
           },
         )
         newAttachments.push(...data)

packages/nocodb-sdk/src/lib/Api.ts

@@ -85,6 +85,26 @@ export interface AttachmentReqType {
   title?: string;
   /** Attachment URL to be uploaded via upload-by-url */
   url?: string;
+  /** The name of the attachment file name */
+  fileName?: string;
+}
+
+/**
+ * Model for File Request
+ */
+export interface FileReqType {
+  /** The mimetype of the file */
+  mimetype?: string;
+  /** The name of the input used to upload the file */
+  fieldname?: string;
+  /** The original name of the file */
+  originalname?: string;
+  /** The size of the file */
+  size?: number;
+  /** The encoding of the file */
+  encoding?: string;
+  /** An buffer array containing the file content */
+  buffer?: any;
 }
 
 /**
@@ -10308,7 +10328,9 @@ export class Api<
          */
         path: string;
       },
-      data: AttachmentReqType,
+      data: {
+        files: FileReqType[];
+      },
       params: RequestParams = {}
     ) =>
       this.request<any, any>({

packages/nocodb-sdk/src/lib/CustomAPI.ts

@@ -152,6 +152,17 @@ export interface DataListPayloadType {
   filters?: FilterType[];
 }
 
+export interface FileType {
+  fieldname: string;
+  originalname: string;
+  encoding: string;
+  mimetype: string;
+  destination: string;
+  filename: string;
+  size: number;
+  path: string;
+}
+
 export interface DataListParamsType {
   limit?: string;
   offset?: string;

packages/nocodb-sdk/src/lib/index.ts

@@ -14,7 +14,7 @@ export {
   isVirtualCol,
   isLinksOrLTAR,
 } from '~/lib/UITypes';
-export { default as CustomAPI } from '~/lib/CustomAPI';
+export { default as CustomAPI, FileType } from '~/lib/CustomAPI';
 export { default as TemplateGenerator } from '~/lib/TemplateGenerator';
 export * from '~/lib/passwordHelpers';
 export * from '~/lib/mergeSwaggerSchema';

packages/nocodb/src/controllers/attachments-secure.controller.ts

@@ -15,6 +15,9 @@ import {
 import hash from 'object-hash';
 import moment from 'moment';
 import { AnyFilesInterceptor } from '@nestjs/platform-express';
+import { Response as ResponseType } from 'express';
+import type { Request as RequestType } from 'express';
+import type { AttachmentReqType, FileType } from 'nocodb-sdk';
 import { GlobalGuard } from '~/guards/global/global.guard';
 import { AttachmentsService } from '~/services/attachments.service';
 import { PresignedUrl } from '~/models';
@@ -29,7 +32,10 @@ export class AttachmentsSecureController {
   @Post(['/api/v1/db/storage/upload', '/api/v2/storage/upload'])
   @HttpCode(200)
   @UseInterceptors(UploadAllowedInterceptor, AnyFilesInterceptor())
-  async upload(@UploadedFiles() files: Array<any>, @Request() req) {
+  async upload(
+    @UploadedFiles() files: Array<FileType>,
+    @Request() req: RequestType & { user: { id: string } },
+  ) {
     const path = `${moment().format('YYYY/MM/DD')}/${hash(req.user.id)}`;
 
     const attachments = await this.attachmentsService.upload({
@@ -44,7 +50,10 @@ export class AttachmentsSecureController {
   @HttpCode(200)
   @UseInterceptors(UploadAllowedInterceptor)
   @UseGuards(MetaApiLimiterGuard, GlobalGuard)
-  async uploadViaURL(@Body() body: any, @Request() req) {
+  async uploadViaURL(
+    @Body() body: Array<AttachmentReqType>,
+    @Request() req: RequestType & { user: { id: string } },
+  ) {
     const path = `${moment().format('YYYY/MM/DD')}/${hash(req.user.id)}`;
 
     const attachments = await this.attachmentsService.uploadViaURL({
@@ -56,15 +65,18 @@ export class AttachmentsSecureController {
   }
 
   @Get('/dltemp/:param(*)')
-  async fileReadv3(@Param('param') param: string, @Response() res) {
+  async fileReadv3(
+    @Param('param') param: string,
+    @Response() res: ResponseType,
+  ) {
     try {
       const fpath = await PresignedUrl.getPath(`dltemp/${param}`);
 
-      const { img } = await this.attachmentsService.fileRead({
+      const file = await this.attachmentsService.getFile({
         path: path.join('nc', 'uploads', fpath),
       });
 
-      res.sendFile(img);
+      res.sendFile(file.path);
     } catch (e) {
       res.status(404).send('Not found');
     }

packages/nocodb/src/controllers/attachments.controller.ts

@@ -14,6 +14,8 @@ import {
   UseInterceptors,
 } from '@nestjs/common';
 import { AnyFilesInterceptor } from '@nestjs/platform-express';
+import { Request as RequestType, Response as ResponseType } from 'express';
+import type { AttachmentReqType, FileType } from 'nocodb-sdk';
 import { UploadAllowedInterceptor } from '~/interceptors/is-upload-allowed/is-upload-allowed.interceptor';
 import { GlobalGuard } from '~/guards/global/global.guard';
 import { AttachmentsService } from '~/services/attachments.service';
@@ -29,13 +31,12 @@ export class AttachmentsController {
   @HttpCode(200)
   @UseInterceptors(UploadAllowedInterceptor, AnyFilesInterceptor())
   async upload(
-    @UploadedFiles() files: Array<any>,
-    @Body() body: any,
-    @Request() req: any,
+    @UploadedFiles() files: Array<FileType>,
+    @Request() req: RequestType,
   ) {
     const attachments = await this.attachmentsService.upload({
       files: files,
-      path: req.query?.path as string,
+      path: req.query?.path?.toString(),
     });
 
     return attachments;
@@ -45,7 +46,10 @@ export class AttachmentsController {
   @HttpCode(200)
   @UseInterceptors(UploadAllowedInterceptor)
   @UseGuards(MetaApiLimiterGuard, GlobalGuard)
-  async uploadViaURL(@Body() body: any, @Query('path') path: string) {
+  async uploadViaURL(
+    @Body() body: Array<AttachmentReqType>,
+    @Query('path') path: string,
+  ) {
     const attachments = await this.attachmentsService.uploadViaURL({
       urls: body,
       path,
@@ -58,16 +62,17 @@ export class AttachmentsController {
   // , getCacheMiddleware(), catchError(fileRead));
   @Get('/download/:filename(*)')
   // This route will match any URL that starts with
-  async fileRead(@Param('filename') filename: string, @Response() res) {
+  async fileRead(
+    @Param('filename') filename: string,
+    @Response() res: ResponseType,
+  ) {
     try {
-      const { img, type } = await this.attachmentsService.fileRead({
+      const file = await this.attachmentsService.getFile({
         path: path.join('nc', 'uploads', filename),
       });
 
-      res.writeHead(200, { 'Content-Type': type });
-      res.end(img, 'binary');
+      res.sendFile(file.path);
     } catch (e) {
-      console.log(e);
       res.status(404).send('Not found');
     }
   }
@@ -79,10 +84,10 @@ export class AttachmentsController {
     @Param('param1') param1: string,
     @Param('param2') param2: string,
     @Param('filename') filename: string,
-    @Response() res,
+    @Response() res: ResponseType,
   ) {
     try {
-      const { img, type } = await this.attachmentsService.fileRead({
+      const file = await this.attachmentsService.getFile({
         path: path.join(
           'nc',
           param1,
@@ -92,23 +97,25 @@ export class AttachmentsController {
         ),
       });
 
-      res.writeHead(200, { 'Content-Type': type });
-      res.end(img, 'binary');
+      res.sendFile(file.path);
     } catch (e) {
       res.status(404).send('Not found');
     }
   }
 
   @Get('/dltemp/:param(*)')
-  async fileReadv3(@Param('param') param: string, @Response() res) {
+  async fileReadv3(
+    @Param('param') param: string,
+    @Response() res: ResponseType,
+  ) {
     try {
       const fpath = await PresignedUrl.getPath(`dltemp/${param}`);
 
-      const { img } = await this.attachmentsService.fileRead({
+      const file = await this.attachmentsService.getFile({
         path: path.join('nc', 'uploads', fpath),
       });
 
-      res.sendFile(img);
+      res.sendFile(file.path);
     } catch (e) {
       res.status(404).send('Not found');
     }

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -4312,10 +4312,7 @@ class BaseModelSqlv2 {
         }
       }
 
-      if (
-        this.isPg &&
-        (col.dt === 'timestamp with time zone' || col.dt === 'timestamptz')
-      ) {
+      if (this.isPg) {
         // postgres - timezone already attached to input
         // e.g. 2023-05-11 16:16:51+08:00
         keepLocalTime = false;

packages/nocodb/src/helpers/webhookHelpers.ts

@@ -2,6 +2,7 @@ import Handlebars from 'handlebars';
 import { v4 as uuidv4 } from 'uuid';
 import axios from 'axios';
 import { useAgent } from 'request-filtering-agent';
+import { Logger } from '@nestjs/common';
 import NcPluginMgrv2 from './NcPluginMgrv2';
 import type { Column, FormView, Hook, Model, View } from '~/models';
 import type { HookLogType } from 'nocodb-sdk';
@@ -11,6 +12,8 @@ Handlebars.registerHelper('json', function (context) {
   return JSON.stringify(context);
 });
 
+const logger = new Logger('webhookHelpers');
+
 export function parseBody(template: string, data: any): string {
   if (!template) {
     return template;
@@ -409,7 +412,16 @@ export async function invokeWebhook(
         break;
     }
   } catch (e) {
-    console.log(e);
+    if (e.response) {
+      logger.log({
+        data: e.response.data,
+        status: e.response.status,
+        url: e.response.config?.url,
+        message: e.message,
+      });
+    } else {
+      logger.log(e.message, e.stack);
+    }
     if (['ERROR', 'ALL'].includes(process.env.NC_AUTOMATION_LOG_LEVEL)) {
       hookLog = {
         ...hook,

packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@@ -78,15 +78,20 @@ export class ExportService {
           if (source.type === 'pg') {
             if (column.ai) {
               try {
+                const baseModel = await Model.getBaseModelSQL({
+                  id: model.id,
+                  viewId: null,
+                  dbDriver: await NcConnectionMgrv2.get(source),
+                });
                 const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
-                const seq = await sqlClient.knex.raw(
+                const seq = await sqlClient.raw(
                   `SELECT pg_get_serial_sequence('??', ?) as seq;`,
-                  [model.table_name, column.column_name],
+                  [baseModel.getTnPath(model.table_name), column.column_name],
                 );
                 if (seq.rows.length > 0) {
                   const seqName = seq.rows[0].seq;
 
-                  const res = await sqlClient.knex.raw(
+                  const res = await sqlClient.raw(
                     `SELECT last_value as last FROM ${seqName};`,
                   );
 

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -158,10 +158,19 @@ export class ImportService {
         if (source.type === 'pg') {
           if (modelData.pgSerialLastVal) {
             if (col.ai) {
+              const baseModel = await Model.getBaseModelSQL({
+                id: table.id,
+                viewId: null,
+                dbDriver: await NcConnectionMgrv2.get(source),
+              });
               const sqlClient = await NcConnectionMgrv2.getSqlClient(source);
-              await sqlClient.knex.raw(
+              await sqlClient.raw(
                 `SELECT setval(pg_get_serial_sequence('??', ?), ?);`,
-                [table.table_name, col.column_name, modelData.pgSerialLastVal],
+                [
+                  baseModel.getTnPath(table.table_name),
+                  col.column_name,
+                  modelData.pgSerialLastVal,
+                ],
               );
             }
           }

packages/nocodb/src/schema/swagger.json

@@ -14939,15 +14939,30 @@
           "content": {
             "multipart/form-data": {
               "schema": {
-                "$ref": "#/components/schemas/AttachmentReq"
+                "type":"object",
+                "properties":{
+                  "files":{
+                    "type": "array",
+                    "required": true,
+                    "items": {
+                      "$ref": "#/components/schemas/FileReq"
+                    }
+                  }
+                }
               },
               "examples": {
                 "Example 1": {
                   "value": {
-                    "mimetype": "image/jpeg",
-                    "path": "download/noco/jango_fett/Table1/attachment/uVbjPVQxC_SSfs8Ctx.jpg",
-                    "size": 13052,
-                    "title": "22bc-kavypmq4869759 (1).jpg"
+                    "files": [
+                      {
+                        "mimetype": "image/jpeg",
+                        "fieldname":"files",
+                        "originalname": "22bc-kavypmq4869759 (1).jpg",
+                        "encoding":"7bit",
+                        "size": 13052,
+                        "buffer": "<Buffer 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 03 20 00 00 02 58 08 02 00 00 00 15 14 15 27 00 00 00 09 70 48 59 73 00 00 0b 13 00 00 0b 13 01 >"
+                      }
+                    ]
                   }
                 }
               }
@@ -16478,12 +16493,59 @@
           "url": {
             "type": "string",
             "description": "Attachment URL to be uploaded via upload-by-url"
+          },
+          "fileName":{
+            "type": "string",
+            "description": "The name of the attachment file name"
           }
         },
         "x-stoplight": {
           "id": "6cr1iwhbyxncd"
         }
       },
+      "FileReq": {
+        "description": "Model for File Request",
+        "type": "object",
+        "x-examples": {
+          "Example 1": {
+            "mimetype": "image/jpeg",
+            "fieldname":"files",
+            "originalname": "22bc-kavypmq4869759 (1).jpg",
+            "encoding":"7bit",
+            "size": 13052,
+            "buffer": "<Buffer 89 50 4e 47 0d 0a 1a 0a 00 00 00 0d 49 48 44 52 00 00 03 20 00 00 02 58 08 02 00 00 00 15 14 15 27 00 00 00 09 70 48 59 73 00 00 0b 13 00 00 0b 13 01 >"
+          }
+        },
+        "title": "File Request Model",
+        "properties": {
+          "mimetype": {
+            "type": "string",
+            "description": "The mimetype of the file"
+          },
+          "fieldname": {
+            "type": "string",
+            "description": "The name of the input used to upload the file"
+          },
+          "originalname": {
+            "type": "string",
+            "description": "The original name of the file"
+          },
+          "size": {
+            "type": "number",
+            "description": "The size of the file"
+          },
+          "encoding": {
+            "type": "string",
+            "description": "The encoding of the file"
+          },
+          "buffer": {
+            "description": "An buffer array containing the file content"
+          }
+        },
+        "x-stoplight": {
+          "id": null
+        }
+      },
       "Audit": {
         "description": "Model for Audit",
         "examples": [

packages/nocodb/src/services/attachments.service.ts

@@ -3,6 +3,7 @@ import { AppEvents } from 'nocodb-sdk';
 import { Injectable } from '@nestjs/common';
 import { nanoid } from 'nanoid';
 import slash from 'slash';
+import type { AttachmentReqType, FileType } from 'nocodb-sdk';
 import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
 import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
 import Local from '~/plugins/storage/Local';
@@ -14,11 +15,7 @@ import { utf8ify } from '~/helpers/stringHelpers';
 export class AttachmentsService {
   constructor(private readonly appHooksService: AppHooksService) {}
 
-  async upload(param: {
-    path?: string;
-    // todo: proper type
-    files: unknown[];
-  }) {
+  async upload(param: { path?: string; files: FileType[] }) {
     // TODO: add getAjvValidatorMw
     const filePath = this.sanitizeUrlPath(
       param.path?.toString()?.split('/') || [''],
@@ -28,7 +25,7 @@ export class AttachmentsService {
     const storageAdapter = await NcPluginMgrv2.storageAdapter();
 
     const attachments = await Promise.all(
-      param.files?.map(async (file: any) => {
+      param.files?.map(async (file) => {
         const originalName = utf8ify(file.originalname);
         const fileName = `${nanoid(18)}${path.extname(originalName)}`;
 
@@ -91,15 +88,7 @@ export class AttachmentsService {
     return attachments;
   }
 
-  async uploadViaURL(param: {
-    path?: string;
-    urls: {
-      url: string;
-      fileName: string;
-      mimetype?: string;
-      size?: string | number;
-    }[];
-  }) {
+  async uploadViaURL(param: { path?: string; urls: AttachmentReqType[] }) {
     // TODO: add getAjvValidatorMw
     const filePath = this.sanitizeUrlPath(
       param?.path?.toString()?.split('/') || [''],
@@ -116,12 +105,13 @@ export class AttachmentsService {
           _fileName || url.split('/').pop(),
         )}`;
 
-        const attachmentUrl = await (storageAdapter as any).fileCreateByUrl(
-          slash(path.join(destPath, fileName)),
-          url,
-        );
+        const attachmentUrl: string | null =
+          await storageAdapter.fileCreateByUrl(
+            slash(path.join(destPath, fileName)),
+            url,
+          );
 
-        let attachmentPath;
+        let attachmentPath: string | undefined;
 
         // if `attachmentUrl` is null, then it is local attachment
         if (!attachmentUrl) {
@@ -147,18 +137,21 @@ export class AttachmentsService {
     return attachments;
   }
 
-  async fileRead(param: { path: string }) {
+  async getFile(param: { path: string }): Promise<{
+    path: string;
+    type: string;
+  }> {
     // get the local storage adapter to display local attachments
     const storageAdapter = new Local();
     const type =
       mimetypes[path.extname(param.path).split('/').pop().slice(1)] ||
       'text/plain';
 
-    const img = await storageAdapter.validateAndNormalisePath(
+    const filePath = await storageAdapter.validateAndNormalisePath(
       slash(param.path),
       true,
     );
-    return { img, type };
+    return { path: filePath, type };
   }
 
   sanitizeUrlPath(paths) {