From 1b299ab8b26b77fa192d0ef1812cebe6f054534b Mon Sep 17 00:00:00 2001 From: mertmit Date: Sat, 28 Oct 2023 10:19:47 +0300 Subject: [PATCH 1/6] fix: use sendFile for old apis Signed-off-by: mertmit --- .../controllers/attachments-secure.controller.ts | 4 ++-- .../src/controllers/attachments.controller.ts | 15 ++++++--------- .../nocodb/src/services/attachments.service.ts | 9 ++++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/nocodb/src/controllers/attachments-secure.controller.ts b/packages/nocodb/src/controllers/attachments-secure.controller.ts index dc04954946..40b6a6ebf6 100644 --- a/packages/nocodb/src/controllers/attachments-secure.controller.ts +++ b/packages/nocodb/src/controllers/attachments-secure.controller.ts @@ -60,11 +60,11 @@ export class AttachmentsSecureController { try { const fpath = await PresignedUrl.getPath(`dltemp/${param}`); - const { img } = await this.attachmentsService.fileRead({ + const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', fpath), }); - res.sendFile(img); + res.sendFile(file.path); } catch (e) { res.status(404).send('Not found'); } diff --git a/packages/nocodb/src/controllers/attachments.controller.ts b/packages/nocodb/src/controllers/attachments.controller.ts index 8b75a228d5..e5d4caf4de 100644 --- a/packages/nocodb/src/controllers/attachments.controller.ts +++ b/packages/nocodb/src/controllers/attachments.controller.ts @@ -60,14 +60,12 @@ export class AttachmentsController { // This route will match any URL that starts with async fileRead(@Param('filename') filename: string, @Response() res) { try { - const { img, type } = await this.attachmentsService.fileRead({ + const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', filename), }); - res.writeHead(200, { 'Content-Type': type }); - res.end(img, 'binary'); + res.sendFile(file.path); } catch (e) { - console.log(e); res.status(404).send('Not found'); } } @@ -82,7 +80,7 @@ export class AttachmentsController { @Response() res, ) { try { - const { img, type } = await this.attachmentsService.fileRead({ + const file = await this.attachmentsService.getFile({ path: path.join( 'nc', param1, @@ -92,8 +90,7 @@ export class AttachmentsController { ), }); - res.writeHead(200, { 'Content-Type': type }); - res.end(img, 'binary'); + res.sendFile(file.path); } catch (e) { res.status(404).send('Not found'); } @@ -104,11 +101,11 @@ export class AttachmentsController { try { const fpath = await PresignedUrl.getPath(`dltemp/${param}`); - const { img } = await this.attachmentsService.fileRead({ + const file = await this.attachmentsService.getFile({ path: path.join('nc', 'uploads', fpath), }); - res.sendFile(img); + res.sendFile(file.path); } catch (e) { res.status(404).send('Not found'); } diff --git a/packages/nocodb/src/services/attachments.service.ts b/packages/nocodb/src/services/attachments.service.ts index c8e835e61f..2b9ba3eed6 100644 --- a/packages/nocodb/src/services/attachments.service.ts +++ b/packages/nocodb/src/services/attachments.service.ts @@ -146,18 +146,21 @@ export class AttachmentsService { return attachments; } - async fileRead(param: { path: string }) { + async getFile(param: { path: string }): Promise<{ + path: string; + type: string; + }> { // get the local storage adapter to display local attachments const storageAdapter = new Local(); const type = mimetypes[path.extname(param.path).split('/').pop().slice(1)] || 'text/plain'; - const img = await storageAdapter.validateAndNormalisePath( + const filePath = await storageAdapter.validateAndNormalisePath( slash(param.path), true, ); - return { img, type }; + return { path: filePath, type }; } sanitizeUrlPath(paths) { From 2a2f8047af4c23bf50e5cb596006c782e318bf4b Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Nov 2023 05:41:10 +0000 Subject: [PATCH 2/6] refactor: webhook log exclude request object --- packages/nocodb/src/helpers/webhookHelpers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/nocodb/src/helpers/webhookHelpers.ts b/packages/nocodb/src/helpers/webhookHelpers.ts index 3641a84b9b..14cc6f71b4 100644 --- a/packages/nocodb/src/helpers/webhookHelpers.ts +++ b/packages/nocodb/src/helpers/webhookHelpers.ts @@ -2,6 +2,7 @@ import Handlebars from 'handlebars'; import { v4 as uuidv4 } from 'uuid'; import axios from 'axios'; import { useAgent } from 'request-filtering-agent'; +import { Logger } from '@nestjs/common'; import NcPluginMgrv2 from './NcPluginMgrv2'; import type { Column, FormView, Hook, Model, View } from '~/models'; import type { HookLogType } from 'nocodb-sdk'; @@ -11,6 +12,8 @@ Handlebars.registerHelper('json', function (context) { return JSON.stringify(context); }); +const logger = new Logger('webhookHelpers'); + export function parseBody(template: string, data: any): string { if (!template) { return template; @@ -409,7 +412,13 @@ export async function invokeWebhook( break; } } catch (e) { - console.log(e); + if (e.response) { + logger.log(e.response.data); + logger.log(e.response.status); + logger.log(e.response.headers); + } else { + logger.log(e.message, e.stack); + } if (['ERROR', 'ALL'].includes(process.env.NC_AUTOMATION_LOG_LEVEL)) { hookLog = { ...hook, From 8008befe330ec0e819ffe0d5e49f0bbe7619629e Mon Sep 17 00:00:00 2001 From: Pranav C Date: Wed, 1 Nov 2023 05:41:10 +0000 Subject: [PATCH 3/6] refactor: better message --- packages/nocodb/src/helpers/webhookHelpers.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/nocodb/src/helpers/webhookHelpers.ts b/packages/nocodb/src/helpers/webhookHelpers.ts index 14cc6f71b4..6ce9fa32a5 100644 --- a/packages/nocodb/src/helpers/webhookHelpers.ts +++ b/packages/nocodb/src/helpers/webhookHelpers.ts @@ -413,9 +413,12 @@ export async function invokeWebhook( } } catch (e) { if (e.response) { - logger.log(e.response.data); - logger.log(e.response.status); - logger.log(e.response.headers); + logger.log({ + data: e.response.data, + status: e.response.status, + url: e.response.config?.url, + message: e.message, + }); } else { logger.log(e.message, e.stack); } From 1af22332d68a0e8a0f5479f96210f14847542d11 Mon Sep 17 00:00:00 2001 From: mertmit Date: Fri, 3 Nov 2023 15:59:58 +0000 Subject: [PATCH 4/6] fix: this record exist issue for pg duplicate --- .../jobs/jobs/export-import/export.service.ts | 11 ++++++++--- .../jobs/jobs/export-import/import.service.ts | 13 +++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts index 544919cfbc..28a49df122 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts @@ -78,15 +78,20 @@ export class ExportService { if (source.type === 'pg') { if (column.ai) { try { + const baseModel = await Model.getBaseModelSQL({ + id: model.id, + viewId: null, + dbDriver: await NcConnectionMgrv2.get(source), + }); const sqlClient = await NcConnectionMgrv2.getSqlClient(source); - const seq = await sqlClient.knex.raw( + const seq = await sqlClient.raw( `SELECT pg_get_serial_sequence('??', ?) as seq;`, - [model.table_name, column.column_name], + [baseModel.getTnPath(model.table_name), column.column_name], ); if (seq.rows.length > 0) { const seqName = seq.rows[0].seq; - const res = await sqlClient.knex.raw( + const res = await sqlClient.raw( `SELECT last_value as last FROM ${seqName};`, ); diff --git a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts index 4780fea595..43d34032e8 100644 --- a/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts @@ -158,10 +158,19 @@ export class ImportService { if (source.type === 'pg') { if (modelData.pgSerialLastVal) { if (col.ai) { + const baseModel = await Model.getBaseModelSQL({ + id: table.id, + viewId: null, + dbDriver: await NcConnectionMgrv2.get(source), + }); const sqlClient = await NcConnectionMgrv2.getSqlClient(source); - await sqlClient.knex.raw( + await sqlClient.raw( `SELECT setval(pg_get_serial_sequence('??', ?), ?);`, - [table.table_name, col.column_name, modelData.pgSerialLastVal], + [ + baseModel.getTnPath(table.table_name), + col.column_name, + modelData.pgSerialLastVal, + ], ); } } From 8b6350535bddf5c9837c9e85bc0c66bc56461cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D5=A1=C9=A8=D5=BC=C9=A2=D3=84=D5=A1=D6=85=D5=BC=C9=A2?= Date: Sat, 4 Nov 2023 17:38:57 +0800 Subject: [PATCH 5/6] fix(nocodb): pg datetime issue --- packages/nocodb/src/db/BaseModelSqlv2.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/nocodb/src/db/BaseModelSqlv2.ts b/packages/nocodb/src/db/BaseModelSqlv2.ts index b9114ae056..f5ce571808 100644 --- a/packages/nocodb/src/db/BaseModelSqlv2.ts +++ b/packages/nocodb/src/db/BaseModelSqlv2.ts @@ -4304,10 +4304,7 @@ class BaseModelSqlv2 { } } - if ( - this.isPg && - (col.dt === 'timestamp with time zone' || col.dt === 'timestamptz') - ) { + if (this.isPg) { // postgres - timezone already attached to input // e.g. 2023-05-11 16:16:51+08:00 keepLocalTime = false; From 9c5116ac98c22eeec7f3b39ee9021a76d9995c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D5=A1=C9=A8=D5=BC=C9=A2=D3=84=D5=A1=D6=85=D5=BC=C9=A2?= Date: Sun, 5 Nov 2023 13:14:13 +0800 Subject: [PATCH 6/6] chore(sdk): regenerate Api.ts --- packages/nocodb-sdk/src/lib/Api.ts | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/nocodb-sdk/src/lib/Api.ts b/packages/nocodb-sdk/src/lib/Api.ts index 4a080a7941..722bbb1cc5 100644 --- a/packages/nocodb-sdk/src/lib/Api.ts +++ b/packages/nocodb-sdk/src/lib/Api.ts @@ -85,6 +85,26 @@ export interface AttachmentReqType { title?: string; /** Attachment URL to be uploaded via upload-by-url */ url?: string; + /** The name of the attachment file name */ + fileName?: string; +} + +/** + * Model for File Request + */ +export interface FileReqType { + /** The mimetype of the file */ + mimetype?: string; + /** The name of the input used to upload the file */ + fieldname?: string; + /** The original name of the file */ + originalname?: string; + /** The size of the file */ + size?: number; + /** The encoding of the file */ + encoding?: string; + /** An buffer array containing the file content */ + buffer?: any; } /** @@ -10308,7 +10328,9 @@ export class Api< */ path: string; }, - data: AttachmentReqType, + data: { + files: FileReqType[]; + }, params: RequestParams = {} ) => this.request({