Browse Source

Merge pull request #9555 from nocodb/nc-fix/sync-calls

fix: optimizations to potential blockers
pull/9570/head
Pranav C 3 months ago committed by GitHub
parent
commit
b515e2fa3f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 9
      packages/nocodb/src/Noco.ts
  2. 4
      packages/nocodb/src/controllers/attachments-secure.controller.ts
  3. 9
      packages/nocodb/src/controllers/attachments.controller.ts
  4. 8
      packages/nocodb/src/helpers/attachmentHelpers.ts
  5. 15
      packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts
  6. 9
      packages/nocodb/src/modules/jobs/migration-jobs/nc_job_002_thumbnail.ts
  7. 23
      packages/nocodb/src/services/attachments.service.ts

9
packages/nocodb/src/Noco.ts

@ -15,6 +15,7 @@ import type http from 'http';
import { MetaTable, RootScopes } from '~/utils/globals';
import { AppModule } from '~/app.module';
import { isEE, T } from '~/utils';
import type Sharp from 'sharp';
dotenv.config();
@ -43,6 +44,8 @@ export default class Noco {
protected config: any;
protected requestContext: any;
public static sharp: typeof Sharp;
constructor() {
process.env.PORT = process.env.PORT || '8080';
// todo: move
@ -100,6 +103,12 @@ export default class Noco {
this.initCustomLogger(nestApp);
nestApp.flushLogs();
try {
this.sharp = (await import('sharp')).default;
} catch {
console.error('Sharp is not available for your platform, thumbnail generation will be skipped');
}
if (process.env.NC_WORKER_CONTAINER === 'true') {
if (!process.env.NC_REDIS_URL) {
throw new Error('NC_REDIS_URL is required');

4
packages/nocodb/src/controllers/attachments-secure.controller.ts

@ -1,5 +1,4 @@
import path from 'path';
import fs from 'fs';
import {
Body,
Controller,
@ -29,6 +28,7 @@ import { DataApiLimiterGuard } from '~/guards/data-api-limiter.guard';
import { TenantContext } from '~/decorators/tenant-context.decorator';
import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware';
import { NcError } from '~/helpers/catchError';
import { localFileExists } from '~/helpers/attachmentHelpers';
@Controller()
export class AttachmentsSecureController {
@ -95,7 +95,7 @@ export class AttachmentsSecureController {
path: path.join('nc', filePath, fpath),
});
if (!fs.existsSync(file.path)) {
if (!(await localFileExists(file.path))) {
return res.status(404).send('File not found');
}

9
packages/nocodb/src/controllers/attachments.controller.ts

@ -1,5 +1,4 @@
import path from 'path';
import fs from 'fs';
import {
Body,
Controller,
@ -24,7 +23,7 @@ import { AttachmentsService } from '~/services/attachments.service';
import { PresignedUrl } from '~/models';
import { MetaApiLimiterGuard } from '~/guards/meta-api-limiter.guard';
import { NcContext, NcRequest } from '~/interface/config';
import { isPreviewAllowed } from '~/helpers/attachmentHelpers';
import { isPreviewAllowed, localFileExists } from '~/helpers/attachmentHelpers';
import { DataTableService } from '~/services/data-table.service';
import { TenantContext } from '~/decorators/tenant-context.decorator';
import { DataApiLimiterGuard } from '~/guards/data-api-limiter.guard';
@ -85,7 +84,7 @@ export class AttachmentsController {
path: path.join('nc', 'uploads', filename),
});
if (!fs.existsSync(file.path)) {
if (!(await localFileExists(file.path))) {
return res.status(404).send('File not found');
}
@ -126,7 +125,7 @@ export class AttachmentsController {
),
});
if (!fs.existsSync(file.path)) {
if (!(await localFileExists(file.path))) {
return res.status(404).send('File not found');
}
@ -172,7 +171,7 @@ export class AttachmentsController {
path: path.join('nc', filePath, fpath),
});
if (!fs.existsSync(file.path)) {
if (!(await localFileExists(file.path))) {
return res.status(404).send('File not found');
}

8
packages/nocodb/src/helpers/attachmentHelpers.ts

@ -1,4 +1,5 @@
import path from 'path';
import fs from 'fs';
import mime from 'mime/lite';
import slash from 'slash';
import { getToolDir } from '~/utils/nc-config';
@ -64,3 +65,10 @@ export function getPathFromUrl(url: string, removePrefix = false) {
return decodeURI(`${pathName}${newUrl.search}${newUrl.hash}`);
}
export const localFileExists = (path: string) => {
return fs.promises
.access(path)
.then(() => true)
.catch(() => false);
};

15
packages/nocodb/src/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor.ts

@ -6,9 +6,9 @@ import type { IStorageAdapterV2 } from '~/types/nc-plugin';
import type { Job } from 'bull';
import type { AttachmentResType } from 'nocodb-sdk';
import type { ThumbnailGeneratorJobData } from '~/interface/Jobs';
import type Sharp from 'sharp';
import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
import { getPathFromUrl } from '~/helpers/attachmentHelpers';
import Noco from '~/Noco';
export class ThumbnailGeneratorProcessor {
private logger = new Logger(ThumbnailGeneratorProcessor.name);
@ -39,19 +39,10 @@ export class ThumbnailGeneratorProcessor {
private async generateThumbnail(
attachment: AttachmentResType,
): Promise<{ [key: string]: string }> {
let sharp: typeof Sharp;
try {
sharp = (await import('sharp')).default;
} catch {
// ignore
}
const sharp = Noco.sharp;
if (!sharp) {
this.logger.warn(
`Thumbnail generation is not supported in this platform at the moment.`,
);
return;
return null;
}
sharp.concurrency(1);

9
packages/nocodb/src/modules/jobs/migration-jobs/nc_job_002_thumbnail.ts

@ -2,7 +2,6 @@ import path from 'path';
import debug from 'debug';
import { Injectable } from '@nestjs/common';
import PQueue from 'p-queue';
import type Sharp from 'sharp';
import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
import Noco from '~/Noco';
import mimetypes from '~/utils/mimeTypes';
@ -24,13 +23,7 @@ export class ThumbnailMigration {
async job() {
try {
let sharp: typeof Sharp;
try {
sharp = (await import('sharp')).default;
} catch {
// ignore
}
const sharp = Noco.sharp;
if (!sharp) {
this.log(

23
packages/nocodb/src/services/attachments.service.ts

@ -10,7 +10,6 @@ import hash from 'object-hash';
import moment from 'moment';
import type { AttachmentReqType, FileType } from 'nocodb-sdk';
import type { NcRequest } from '~/interface/config';
import type Sharp from 'sharp';
import { AppHooksService } from '~/services/app-hooks/app-hooks.service';
import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2';
import mimetypes, { mimeIcons } from '~/utils/mimeTypes';
@ -21,6 +20,7 @@ import { IJobsService } from '~/modules/jobs/jobs-service.interface';
import { JobTypes } from '~/interface/Jobs';
import { RootScopes } from '~/utils/globals';
import { validateAndNormaliseLocalPath } from '~/helpers/attachmentHelpers';
import Noco from '~/Noco';
interface AttachmentObject {
url?: string;
@ -88,15 +88,7 @@ export class AttachmentsService {
} = {};
if (file.mimetype.includes('image')) {
let sharp: typeof Sharp;
try {
sharp = (await import('sharp')).default;
} catch (e) {
this.logger.warn(
`Thumbnail generation is not supported in this platform at the moment. Error: ${e.message}`,
);
}
const sharp = Noco.sharp;
if (sharp) {
try {
@ -261,12 +253,7 @@ export class AttachmentsService {
} = {};
if (mimeType.includes('image')) {
let sharp: typeof Sharp;
try {
sharp = (await import('sharp')).default;
} catch {
// ignore
}
const sharp = Noco.sharp;
if (sharp) {
try {
@ -281,10 +268,6 @@ export class AttachmentsService {
} catch (e) {
this.logger.error(`${file.path} is not an image file`);
}
} else {
this.logger.warn(
`Thumbnail generation is not supported in this platform at the moment.`,
);
}
}

Loading…
Cancel
Save