chore: various fixes

packages/nc-gui/components/dashboard/settings/data-sources/CreateBase.vue

@@ -295,6 +295,7 @@ const createSource = async () => {
     )
   } catch (e: any) {
     message.error(await extractSdkResponseErrorMsg(e))
+    creatingSource.value = false
   } finally {
     refreshCommandPalette()
   }

packages/nc-gui/components/smartsheet/toolbar/ColumnFilter.vue

@@ -1,8 +1,9 @@
 <script setup lang="ts">
 import type { ColumnType, FilterType } from 'nocodb-sdk'
-import { UITypes } from 'nocodb-sdk'
+import { PlanLimitTypes, UITypes } from 'nocodb-sdk'
 import {
   ActiveViewInj,
+  AllFiltersInj,
   MetaInj,
   ReloadViewDataHookInj,
   comparisonOpList,
@@ -56,6 +57,8 @@ const activeView = inject(ActiveViewInj, ref())
 
 const reloadDataHook = inject(ReloadViewDataHookInj)!
 
+const isPublic = inject(IsPublicInj, ref(false))
+
 const { $e } = useNuxtApp()
 
 const { nestedFilters } = useSmartsheetStoreOrThrow()
@@ -83,6 +86,8 @@ const {
   webHook.value,
 )
 
+const { getPlanLimit } = useWorkspace()
+
 const localNestedFilters = ref()
 
 const wrapperDomRef = ref<HTMLElement>()
@@ -183,13 +188,22 @@ watch(
   },
 )
 
+const allFilters: Ref<Record<string, FilterType[]>> = inject(AllFiltersInj, ref({}))
+
 watch(
   () => nonDeletedFilters.value.length,
   (length: number) => {
+    allFilters.value[parentId?.value ?? 'root'] = [...nonDeletedFilters.value]
     emit('update:filtersLength', length ?? 0)
   },
 )
 
+const filtersCount = computed(() => {
+  return Object.values(allFilters.value).reduce((acc, filters) => {
+    return acc + filters.filter((el) => !el.is_group).length
+  }, 0)
+})
+
 const applyChanges = async (hookId?: string, _nested = false) => {
   await sync(hookId, _nested)
 
@@ -299,6 +313,10 @@ onMounted(() => {
 onMounted(async () => {
   await loadBtLookupTypes()
 })
+
+onBeforeUnmount(() => {
+  if (parentId.value) delete allFilters.value[parentId.value]
+})
 </script>
 
 <template>
@@ -471,23 +489,44 @@ onMounted(async () => {
       </template>
     </div>
 
-    <div ref="addFiltersRowDomRef" class="flex gap-2">
+    <template v-if="isEeUI && !isPublic">
-      <NcButton size="small" type="text" class="!text-brand-500" @click.stop="addFilter()">
+      <div v-if="filtersCount < getPlanLimit(PlanLimitTypes.FILTER_LIMIT)" ref="addFiltersRowDomRef" class="flex gap-2">
-        <div class="flex items-center gap-1">
+        <NcButton size="small" type="text" class="!text-brand-500" @click.stop="addFilter()">
-          <component :is="iconMap.plus" />
+          <div class="flex items-center gap-1">
-          <!-- Add Filter -->
+            <component :is="iconMap.plus" />
-          {{ $t('activity.addFilter') }}
+            <!-- Add Filter -->
-        </div>
+            {{ $t('activity.addFilter') }}
-      </NcButton>
+          </div>
-
+        </NcButton>
-      <NcButton v-if="!webHook && nestedLevel < 5" type="text" size="small" @click.stop="addFilterGroup()">
+
-        <div class="flex items-center gap-1">
+        <NcButton v-if="!webHook && nestedLevel < 5" type="text" size="small" @click.stop="addFilterGroup()">
-          <!-- Add Filter Group -->
+          <div class="flex items-center gap-1">
-          <component :is="iconMap.plus" />
+            <!-- Add Filter Group -->
-          {{ $t('activity.addFilterGroup') }}
+            <component :is="iconMap.plus" />
-        </div>
+            {{ $t('activity.addFilterGroup') }}
-      </NcButton>
+          </div>
-    </div>
+        </NcButton>
+      </div>
+    </template>
+    <template v-else>
+      <div ref="addFiltersRowDomRef" class="flex gap-2">
+        <NcButton size="small" type="text" class="!text-brand-500" @click.stop="addFilter()">
+          <div class="flex items-center gap-1">
+            <component :is="iconMap.plus" />
+            <!-- Add Filter -->
+            {{ $t('activity.addFilter') }}
+          </div>
+        </NcButton>
+
+        <NcButton v-if="!webHook && nestedLevel < 5" type="text" size="small" @click.stop="addFilterGroup()">
+          <div class="flex items-center gap-1">
+            <!-- Add Filter Group -->
+            <component :is="iconMap.plus" />
+            {{ $t('activity.addFilterGroup') }}
+          </div>
+        </NcButton>
+      </div>
+    </template>
     <div
       v-if="!filters.length"
       class="flex flex-row text-gray-400 mt-2"

packages/nc-gui/components/smartsheet/toolbar/ColumnFilterMenu.vue

@@ -1,6 +1,7 @@
 <script setup lang="ts">
 import {
   ActiveViewInj,
+  AllFiltersInj,
   IsLockedInj,
   computed,
   iconMap,
@@ -48,6 +49,10 @@ watch(
 
 const open = ref(false)
 
+const allFilters = ref({})
+
+provide(AllFiltersInj, allFilters)
+
 useMenuCloseOnEsc(open)
 </script>
 

packages/nc-gui/components/smartsheet/toolbar/SortListMenu.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { RelationTypes, UITypes, isLinksOrLTAR, isSystemColumn } from 'nocodb-sdk'
+import { PlanLimitTypes, RelationTypes, UITypes, isLinksOrLTAR, isSystemColumn } from 'nocodb-sdk'
 import type { ColumnType, LinkToAnotherRecordType } from 'nocodb-sdk'
 import {
   ActiveViewInj,
@@ -10,6 +10,7 @@ import {
   getSortDirectionOptions,
   iconMap,
   inject,
+  isEeUI,
   ref,
   useMenuCloseOnEsc,
   useSmartsheetStoreOrThrow,
@@ -21,6 +22,7 @@ const meta = inject(MetaInj, ref())
 const view = inject(ActiveViewInj, ref())
 const isLocked = inject(IsLockedInj, ref(false))
 const reloadDataHook = inject(ReloadViewDataHookInj)
+const isPublic = inject(IsPublicInj, ref(false))
 
 const { eventBus } = useSmartsheetStoreOrThrow()
 
@@ -32,6 +34,8 @@ const showCreateSort = ref(false)
 
 const { isMobileMode } = useGlobal()
 
+const { getPlanLimit } = useWorkspace()
+
 eventBus.on((event) => {
   if (event === SmartsheetStoreEvents.SORT_RELOAD) {
     loadSorts()
@@ -181,13 +185,31 @@ watch(open, () => {
           :trigger="['click']"
           overlay-class-name="nc-toolbar-dropdown"
         >
-          <NcButton v-e="['c:sort:add']" class="!text-brand-500" type="text" size="small" @click.stop="showCreateSort = true">
+          <template v-if="isEeUI && !isPublic">
-            <div class="flex gap-1 items-center">
+            <NcButton
-              <component :is="iconMap.plus" />
+              v-if="sorts.length < getPlanLimit(PlanLimitTypes.SORT_LIMIT)"
-              <!-- Add Sort Option -->
+              v-e="['c:sort:add']"
-              {{ $t('activity.addSort') }}
+              class="!text-brand-500"
-            </div>
+              type="text"
-          </NcButton>
+              size="small"
+              @click.stop="showCreateSort = true"
+            >
+              <div class="flex gap-1 items-center">
+                <component :is="iconMap.plus" />
+                <!-- Add Sort Option -->
+                {{ $t('activity.addSort') }}
+              </div>
+            </NcButton>
+          </template>
+          <template v-else>
+            <NcButton v-e="['c:sort:add']" class="!text-brand-500" type="text" size="small" @click.stop="showCreateSort = true">
+              <div class="flex gap-1 items-center">
+                <component :is="iconMap.plus" />
+                <!-- Add Sort Option -->
+                {{ $t('activity.addSort') }}
+              </div>
+            </NcButton>
+          </template>
           <template #overlay>
             <SmartsheetToolbarCreateSort :is-parent-open="showCreateSort" @created="addSort" />
           </template>

packages/nc-gui/components/workspace/Billing.vue

@@ -1,44 +0,0 @@
-<script lang="ts" setup>
-import { WorkspacePlan } from 'nocodb-sdk'
-import { storeToRefs } from 'pinia'
-import { extractSdkResponseErrorMsg } from '#imports'
-
-const workspaceStore = useWorkspace()
-const { upgradeActiveWorkspace } = workspaceStore
-const { activeWorkspace } = storeToRefs(workspaceStore)
-const isUpgrading = ref(false)
-
-const upgradeWorkspace = async () => {
-  isUpgrading.value = true
-  try {
-    await upgradeActiveWorkspace()
-  } catch (e: any) {
-    message.error(await extractSdkResponseErrorMsg(e))
-  } finally {
-    isUpgrading.value = false
-  }
-}
-</script>
-
-<template>
-  <div class="h-full w-full flex flex-col justify-center items-center">
-    <div class="mt-20 px-8 py-6 flex flex-col justify-center items-center gap-y-8 border-1 border-gray-100 rounded-md">
-      <template v-if="activeWorkspace.plan === WorkspacePlan.FREE">
-        <div class="flex text-xl font-medium">Upgrade your workspace</div>
-
-        <a-button
-          v-e="['c:workspace:settings:upgrade']"
-          type="primary"
-          size="large"
-          class="!rounded-md"
-          :loading="isUpgrading"
-          @click="upgradeWorkspace"
-          >Upgrade
-        </a-button>
-      </template>
-      <template v-else>
-        <div class="flex text-xl font-medium">Your workspace is upgraded</div>
-      </template>
-    </div>
-  </div>
-</template>

packages/nc-gui/context/index.ts

@@ -1,4 +1,4 @@
-import type { ColumnType, TableType, ViewType } from 'nocodb-sdk'
+import type { ColumnType, FilterType, TableType, ViewType } from 'nocodb-sdk'
 import type { ComputedRef, InjectionKey, Ref } from 'vue'
 import type { EventHook } from '@vueuse/core'
 import type { NcProject, PageSidebarNode, Row, TabItem } from '#imports'
@@ -51,3 +51,4 @@ export const TreeViewInj: InjectionKey<{
   contextMenuTarget: { type?: 'base' | 'base' | 'table' | 'main' | 'layout'; value?: any }
 }> = Symbol('tree-view-functions-injection')
 export const JsonExpandInj: InjectionKey<Ref<boolean>> = Symbol('json-expand-injection')
+export const AllFiltersInj: InjectionKey<Ref<Record<string, FilterType[]>>> = Symbol('all-filters-injection')

packages/nc-gui/store/workspace.ts

@@ -207,6 +207,10 @@ export const useWorkspace = defineStore('workspaceStore', () => {
     isWorkspaceLoading.value = isLoading
   }
 
+  const getPlanLimit = (_arg: any) => {
+    return 9999
+  }
+
   return {
     loadWorkspaces,
     workspaces,
@@ -241,6 +245,7 @@ export const useWorkspace = defineStore('workspaceStore', () => {
     lastPopulatedWorkspaceId,
     isWorkspaceSettingsPageOpened,
     workspaceUserCount,
+    getPlanLimit,
   }
 })
 

packages/nocodb-sdk/src/lib/enums.ts

@@ -157,7 +157,9 @@ export enum WorkspaceStatus {
 
 export enum WorkspacePlan {
   FREE = 'free',
-  PAID = 'paid',
+  STANDARD = 'standard',
+  BUSINESS = 'business',
+  BUSINESS_PRO = 'business-pro',
 }
 
 export const RoleLabels = {
@@ -254,3 +256,29 @@ export const OrderedProjectRoles = [
   ProjectRoles.VIEWER,
   ProjectRoles.NO_ACCESS,
 ];
+
+export enum PlanLimitTypes {
+  // PER USER
+  FREE_WORKSPACE_LIMIT = 'FREE_WORKSPACE_LIMIT',
+
+  // PER WORKSPACE
+  WORKSPACE_USER_LIMIT = 'WORKSPACE_USER_LIMIT',
+  WORKSPACE_ROW_LIMIT = 'WORKSPACE_ROW_LIMIT',
+  BASE_LIMIT = 'BASE_LIMIT',
+
+  // PER BASE
+  SOURCE_LIMIT = 'SOURCE_LIMIT',
+
+  // PER BASE
+  TABLE_LIMIT = 'TABLE_LIMIT',
+
+  // PER TABLE
+  COLUMN_LIMIT = 'COLUMN_LIMIT',
+  TABLE_ROW_LIMIT = 'TABLE_ROW_LIMIT',
+  WEBHOOK_LIMIT = 'WEBHOOK_LIMIT',
+  VIEW_LIMIT = 'VIEW_LIMIT',
+
+  // PER VIEW
+  FILTER_LIMIT = 'FILTER_LIMIT',
+  SORT_LIMIT = 'SORT_LIMIT',
+}

packages/nocodb/src/cache/CacheMgr.ts

@@ -6,6 +6,7 @@ export default abstract class CacheMgr {
     value: any,
     seconds: number,
   ): Promise<any>;
+  public abstract incrby(key: string, value: number): Promise<any>;
   public abstract del(key: string): Promise<any>;
   public abstract getAll(pattern: string): Promise<any[]>;
   public abstract delAll(scope: string, pattern: string): Promise<any[]>;

packages/nocodb/src/cache/NocoCache.ts

@@ -38,6 +38,11 @@ export default class NocoCache {
     );
   }
 
+  public static async incrby(key, value): Promise<boolean> {
+    if (this.cacheDisabled) return Promise.resolve(true);
+    return this.client.incrby(`${this.prefix}:${key}`, value);
+  }
+
   public static async get(key, type): Promise<any> {
     if (this.cacheDisabled) {
       if (type === CacheGetType.TYPE_ARRAY) return Promise.resolve([]);

packages/nocodb/src/cache/RedisCacheMgr.ts

@@ -118,6 +118,11 @@ export default class RedisCacheMgr extends CacheMgr {
     }
   }
 
+  // @ts-ignore
+  async incrby(key: string, value = 1): Promise<any> {
+    return this.client.incrby(key, value);
+  }
+
   // @ts-ignore
   async getAll(pattern: string): Promise<any> {
     return this.client.hgetall(pattern);

packages/nocodb/src/cache/RedisMockCacheMgr.ts

@@ -117,6 +117,11 @@ export default class RedisMockCacheMgr extends CacheMgr {
     }
   }
 
+  // @ts-ignore
+  async incrby(key: string, value = 1): Promise<any> {
+    return this.client.incrby(key, value);
+  }
+
   // @ts-ignore
   async getAll(pattern: string): Promise<any> {
     return this.client.hgetall(pattern);

packages/nocodb/src/db/BaseModelSqlv2.ts

@@ -2804,6 +2804,10 @@ class BaseModelSqlv2 {
         }
       }
 
+      if ('beforeBulkInsert' in this) {
+        await this.beforeBulkInsert(insertDatas, trx, cookie);
+      }
+
       // await this.beforeInsertb(insertDatas, null);
 
       // fallbacks to `10` if database client is sqlite
@@ -3301,6 +3305,10 @@ class BaseModelSqlv2 {
     await this.handleHooks('before.insert', null, data, req);
   }
 
+  public async beforeBulkInsert(data: any, _trx: any, req): Promise<void> {
+    await this.handleHooks('before.bulkInsert', null, data, req);
+  }
+
   public async afterInsert(data: any, _trx: any, req): Promise<void> {
     await this.handleHooks('after.insert', null, data, req);
     const id = this._extractPksValues(data);

packages/nocodb/src/helpers/populateMeta.ts

@@ -187,7 +187,11 @@ export async function extractAndGenerateManyToManyRelations(
   }
 }
 
-export async function populateMeta(source: Source, base: Base): Promise<any> {
+export async function populateMeta(
+  source: Source,
+  base: Base,
+  logger?: (message: string) => void,
+): Promise<any> {
   const info = {
     type: 'rest',
     apiCount: 0,
@@ -253,6 +257,7 @@ export async function populateMeta(source: Source, base: Base): Promise<any> {
   // await this.syncRelations();
 
   const tableMetasInsert = tables.map((table) => {
+    logger?.(`Populating meta for table '${table.title}'`);
     return async () => {
       /* filter relation where this table is present */
       const tableRelations = relations.filter(
@@ -424,6 +429,7 @@ export async function populateMeta(source: Source, base: Base): Promise<any> {
   info.viewsCount = views.length;
 
   const viewMetasInsert = views.map((table) => {
+    logger?.(`Populating meta for view '${table.title}'`);
     return async () => {
       const columns = (
         await sqlClient.columnList({
@@ -479,6 +485,8 @@ export async function populateMeta(source: Source, base: Base): Promise<any> {
 
   (info as any).timeTaken = t2.toFixed(1);
 
+  logger?.(`Populating meta completed in ${t2.toFixed(1)}s`);
+
   return info;
 }
 

packages/nocodb/src/interface/Jobs.ts

@@ -7,6 +7,8 @@ export enum JobTypes {
   MetaSync = 'meta-sync',
   BaseCreate = 'base-create',
   BaseDelete = 'base-delete',
+  UpdateModelStat = 'update-model-stat',
+  UpdateWsStat = 'update-ws-stats',
 }
 
 export enum JobStatus {

packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts

@@ -1,12 +1,12 @@
 import { Injectable } from '@nestjs/common';
 import PQueue from 'p-queue';
 import Emittery from 'emittery';
-import { DuplicateProcessor } from '../jobs/export-import/duplicate.processor';
+import { DuplicateProcessor } from '~/modules/jobs/jobs/export-import/duplicate.processor';
-import { AtImportProcessor } from '../jobs/at-import/at-import.processor';
+import { AtImportProcessor } from '~/modules/jobs/jobs/at-import/at-import.processor';
-import { MetaSyncProcessor } from '../jobs/meta-sync/meta-sync.processor';
+import { MetaSyncProcessor } from '~/modules/jobs/jobs/meta-sync/meta-sync.processor';
-import { SourceCreateProcessor } from '../jobs/source-create/source-create.processor';
+import { SourceCreateProcessor } from '~/modules/jobs/jobs/source-create/source-create.processor';
-import { SourceDeleteProcessor } from '../jobs/source-delete/source-delete.processor';
+import { SourceDeleteProcessor } from '~/modules/jobs/jobs/source-delete/source-delete.processor';
-import { JobsEventService } from './jobs-event.service';
+import { JobsEventService } from '~/modules/jobs/fallback/jobs-event.service';
 import { JobStatus, JobTypes } from '~/interface/Jobs';
 
 export interface Job {
@@ -25,12 +25,12 @@ export class QueueService {
   static emitter = new Emittery();
 
   constructor(
-    private readonly jobsEventService: JobsEventService,
+    protected readonly jobsEventService: JobsEventService,
-    private readonly duplicateProcessor: DuplicateProcessor,
+    protected readonly duplicateProcessor: DuplicateProcessor,
-    private readonly atImportProcessor: AtImportProcessor,
+    protected readonly atImportProcessor: AtImportProcessor,
-    private readonly metaSyncProcessor: MetaSyncProcessor,
+    protected readonly metaSyncProcessor: MetaSyncProcessor,
-    private readonly sourceCreateProcessor: SourceCreateProcessor,
+    protected readonly sourceCreateProcessor: SourceCreateProcessor,
-    private readonly sourceDeleteProcessor: SourceDeleteProcessor,
+    protected readonly sourceDeleteProcessor: SourceDeleteProcessor,
   ) {
     this.emitter.on(JobStatus.ACTIVE, (data: { job: Job }) => {
       const job = this.queueMemory.find((job) => job.id === data.job.id);

packages/nocodb/src/modules/jobs/fallback/jobs.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { QueueService } from './fallback-queue.service';
+import { QueueService } from '~/modules/jobs/fallback/fallback-queue.service';
 import { JobStatus } from '~/interface/Jobs';
 
 @Injectable()

packages/nocodb/src/modules/jobs/helpers.ts

@@ -1,6 +1,8 @@
-import { Logger } from '@nestjs/common';
+import debug from 'debug';
 import { JOBS_QUEUE } from '~/interface/Jobs';
 
+const debugLog = debug('nc:jobs:timings');
+
 export const initTime = function () {
   return {
     hrTime: process.hrtime(),
@@ -15,7 +17,7 @@ export const elapsedTime = function (
   const elapsedS = process.hrtime(time.hrTime)[0].toFixed(3);
   const elapsedMs = process.hrtime(time.hrTime)[1] / 1000000;
   if (label)
-    Logger.debug(
+    debugLog(
       `${label}: ${elapsedS}s ${elapsedMs}ms`,
       `${JOBS_QUEUE}${context ? `:${context}` : ''}`,
     );

packages/nocodb/src/modules/jobs/jobs.controller.ts

@@ -222,7 +222,7 @@ export class JobsController implements OnModuleInit {
       });
     }
 
-    if (process.env.NC_WORKER_CONTAINER === 'true' && this.jobsRedisService) {
+    if (this.jobsRedisService) {
       this.jobsRedisService.publish(jobId, {
         cmd: JobEvents.STATUS,
         ...data,
@@ -292,7 +292,7 @@ export class JobsController implements OnModuleInit {
       });
     }
 
-    if (process.env.NC_WORKER_CONTAINER === 'true' && this.jobsRedisService) {
+    if (this.jobsRedisService) {
       this.jobsRedisService.publish(jobId, {
         cmd: JobEvents.LOG,
         ...data,

packages/nocodb/src/modules/jobs/jobs.module.ts

@@ -32,7 +32,7 @@ import { MetasModule } from '~/modules/metas/metas.module';
 import { DatasModule } from '~/modules/datas/datas.module';
 import { GlobalModule } from '~/modules/global/global.module';
 
-@Module({
+export const JobsModuleMetadata = {
   imports: [
     forwardRef(() => GlobalModule),
     DatasModule,
@@ -81,5 +81,7 @@ import { GlobalModule } from '~/modules/global/global.module';
     SourceDeleteProcessor,
   ],
   exports: ['JobsService'],
-})
+};
+
+@Module(JobsModuleMetadata)
 export class JobsModule {}

packages/nocodb/src/modules/jobs/jobs/at-import/at-import.processor.ts

@@ -86,7 +86,7 @@ const selectColors = {
 
 @Processor(JOBS_QUEUE)
 export class AtImportProcessor {
-  private readonly dubugLog = debug('nc:at-import:processor');
+  private readonly debugLog = debug('nc:jobs:at-import');
 
   constructor(
     private readonly tablesService: TablesService,
@@ -108,6 +108,8 @@ export class AtImportProcessor {
 
   @Process(JobTypes.AtImport)
   async job(job: Job) {
+    this.debugLog(`job started for ${job.id}`);
+
     const syncDB = job.data;
 
     const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
@@ -140,12 +142,12 @@ export class AtImportProcessor {
 
     const logBasic = (log) => {
       this.jobsLogService.sendLog(job, { message: log });
-      this.dubugLog(log);
+      this.debugLog(log);
     };
 
     const logDetailed = (log) => {
       if (debugMode) this.jobsLogService.sendLog(job, { message: log });
-      this.dubugLog(log);
+      this.debugLog(log);
     };
 
     const perfStats = [];
@@ -2435,6 +2437,8 @@ export class AtImportProcessor {
       }
       throw e;
     }
+
+    this.debugLog(`job completed for ${job.id}`);
   }
 }
 

packages/nocodb/src/modules/jobs/jobs/export-import/duplicate.processor.ts

@@ -2,7 +2,7 @@ import { Readable } from 'stream';
 import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
 import papaparse from 'papaparse';
-import { Logger } from '@nestjs/common';
+import debug from 'debug';
 import { isLinksOrLTAR } from 'nocodb-sdk';
 import { Base, Column, Model, Source } from '~/models';
 import { BasesService } from '~/services/bases.service';
@@ -15,9 +15,7 @@ import { ImportService } from '~/modules/jobs/jobs/export-import/import.service'
 
 @Processor(JOBS_QUEUE)
 export class DuplicateProcessor {
-  private readonly logger = new Logger(
+  private readonly debugLog = debug('nc:jobs:duplicate');
-    `${JOBS_QUEUE}:${DuplicateProcessor.name}`,
-  );
 
   constructor(
     private readonly exportService: ExportService,
@@ -28,6 +26,8 @@ export class DuplicateProcessor {
 
   @Process(JobTypes.DuplicateBase)
   async duplicateBase(job: Job) {
+    this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateBase})`);
+
     const hrTime = initTime();
 
     const { baseId, sourceId, dupProjectId, req, options } = job.data;
@@ -114,10 +114,14 @@ export class DuplicateProcessor {
       }
       throw e;
     }
+
+    this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateBase})`);
   }
 
   @Process(JobTypes.DuplicateModel)
   async duplicateModel(job: Job) {
+    this.debugLog(`job started for ${job.id} (${JobTypes.DuplicateModel})`);
+
     const hrTime = initTime();
 
     const { baseId, sourceId, modelId, title, req, options } = job.data;
@@ -216,6 +220,8 @@ export class DuplicateProcessor {
       elapsedTime(hrTime, 'import model data', 'duplicateModel');
     }
 
+    this.debugLog(`job completed for ${job.id} (${JobTypes.DuplicateModel})`);
+
     return await Model.get(findWithIdentifier(idMap, sourceModel.id));
   }
 
@@ -264,7 +270,7 @@ export class DuplicateProcessor {
           handledMmList: handledLinks,
         })
         .catch((e) => {
-          this.logger.error(e);
+          this.debugLog(e);
           dataStream.push(null);
           linkStream.push(null);
           error = e;
@@ -324,7 +330,7 @@ export class DuplicateProcessor {
             _fieldIds: fields,
           })
           .catch((e) => {
-            this.logger.error(e);
+            this.debugLog(e);
             dataStream.push(null);
             linkStream.push(null);
             error = e;
@@ -358,18 +364,18 @@ export class DuplicateProcessor {
                           headers.push(childCol.column_name);
                         } else {
                           headers.push(null);
-                          this.logger.error(`child column not found (${id})`);
+                          this.debugLog(`child column not found (${id})`);
                         }
                       } else {
                         headers.push(col.column_name);
                       }
                     } else {
                       headers.push(null);
-                      this.logger.error(`column not found (${id})`);
+                      this.debugLog(`column not found (${id})`);
                     }
                   } else {
                     headers.push(null);
-                    this.logger.error(`id not found (${header})`);
+                    this.debugLog(`id not found (${header})`);
                   }
                 }
                 parser.resume();
@@ -395,7 +401,7 @@ export class DuplicateProcessor {
                         raw: true,
                       });
                     } catch (e) {
-                      this.logger.error(e);
+                      this.debugLog(e);
                     }
                     chunk = [];
                     parser.resume();
@@ -414,7 +420,7 @@ export class DuplicateProcessor {
                     raw: true,
                   });
                 } catch (e) {
-                  this.logger.error(e);
+                  this.debugLog(e);
                 }
                 chunk = [];
               }

packages/nocodb/src/modules/jobs/jobs/export-import/export.service.ts

@@ -1,7 +1,8 @@
 import { Readable } from 'stream';
 import { isLinksOrLTAR, UITypes, ViewTypes } from 'nocodb-sdk';
 import { unparse } from 'papaparse';
-import { Injectable, Logger } from '@nestjs/common';
+import debug from 'debug';
+import { Injectable } from '@nestjs/common';
 import { elapsedTime, initTime } from '../../helpers';
 import type { BaseModelSqlv2 } from '~/db/BaseModelSqlv2';
 import type { View } from '~/models';
@@ -15,7 +16,7 @@ import { Base, Hook, Model, Source } from '~/models';
 
 @Injectable()
 export class ExportService {
-  private readonly logger = new Logger(ExportService.name);
+  private readonly debugLog = debug('nc:jobs:import');
 
   constructor(private datasService: DatasService) {}
 
@@ -94,7 +95,7 @@ export class ExportService {
                   }
                 }
               } catch (e) {
-                this.logger.error(e);
+                this.debugLog(e);
               }
             }
           }
@@ -457,7 +458,7 @@ export class ExportService {
         true,
       );
     } catch (e) {
-      this.logger.error(e);
+      this.debugLog(e);
       throw e;
     }
 
@@ -525,7 +526,7 @@ export class ExportService {
             true,
           );
         } catch (e) {
-          this.logger.error(e);
+          this.debugLog(e);
           throw e;
         }
 
@@ -714,7 +715,7 @@ export class ExportService {
           });
 
           linkStream.on('error', (e) => {
-            this.logger.error(e);
+            this.debugLog(e);
             resolve(null);
           });
         });
@@ -733,7 +734,7 @@ export class ExportService {
           modelId: model.id,
           handledMmList,
         }).catch((e) => {
-          this.logger.error(e);
+          this.debugLog(e);
           dataStream.push(null);
           linkStream.push(null);
           error = e;

packages/nocodb/src/modules/jobs/jobs/export-import/import.service.ts

@@ -1,6 +1,7 @@
 import { UITypes, ViewTypes } from 'nocodb-sdk';
-import { Injectable, Logger } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import papaparse from 'papaparse';
+import debug from 'debug';
 import { isLinksOrLTAR } from 'nocodb-sdk';
 import { elapsedTime, initTime } from '../../helpers';
 import type { Readable } from 'stream';
@@ -36,7 +37,7 @@ import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2';
 
 @Injectable()
 export class ImportService {
-  private readonly logger = new Logger(ImportService.name);
+  private readonly debugLog = debug('nc:jobs:import');
 
   constructor(
     private tablesService: TablesService,
@@ -1227,7 +1228,7 @@ export class ImportService {
 
               const model = await Model.get(modelId);
 
-              this.logger.debug(`Importing ${model.title}...`);
+              this.debugLog(`Importing ${model.title}...`);
 
               await this.importDataFromCsvStream({
                 idMap,
@@ -1309,7 +1310,7 @@ export class ImportService {
                       headers.push(childCol.column_name);
                     } else {
                       headers.push(null);
-                      this.logger.error(
+                      this.debugLog(
                         `child column not found (${col.colOptions.fk_child_column_id})`,
                       );
                     }
@@ -1318,11 +1319,11 @@ export class ImportService {
                   }
                 } else {
                   headers.push(null);
-                  this.logger.error(`column not found (${id})`);
+                  this.debugLog(`column not found (${id})`);
                 }
               } else {
                 headers.push(null);
-                this.logger.error(`id not found (${header})`);
+                this.debugLog(`id not found (${header})`);
               }
             }
             parser.resume();
@@ -1350,7 +1351,7 @@ export class ImportService {
                     raw: true,
                   });
                 } catch (e) {
-                  this.logger.error(e);
+                  this.debugLog(e);
                 }
                 chunk = [];
                 parser.resume();
@@ -1371,7 +1372,7 @@ export class ImportService {
                 raw: true,
               });
             } catch (e) {
-              this.logger.error(e);
+              this.debugLog(e);
             }
             chunk = [];
           }
@@ -1408,7 +1409,7 @@ export class ImportService {
           });
           lChunks[k] = [];
         } catch (e) {
-          this.logger.error(e);
+          this.debugLog(e);
         }
       }
     };
@@ -1491,7 +1492,7 @@ export class ImportService {
                       [mm.child]: child,
                     });
                   } else {
-                    this.logger.error(`column not found (${columnId})`);
+                    this.debugLog(`column not found (${columnId})`);
                   }
 
                   parser.resume();

packages/nocodb/src/modules/jobs/jobs/meta-sync/meta-sync.processor.ts

@@ -6,12 +6,14 @@ import { MetaDiffsService } from '~/services/meta-diffs.service';
 
 @Processor(JOBS_QUEUE)
 export class MetaSyncProcessor {
-  private readonly debugLog = debug('nc:meta-sync:processor');
+  private readonly debugLog = debug('nc:jobs:meta-sync');
 
   constructor(private readonly metaDiffsService: MetaDiffsService) {}
 
   @Process(JobTypes.MetaSync)
   async job(job: Job) {
+    this.debugLog(`job started for ${job.id}`);
+
     const info: {
       baseId: string;
       sourceId: string;
@@ -26,5 +28,7 @@ export class MetaSyncProcessor {
         sourceId: info.sourceId,
       });
     }
+
+    this.debugLog(`job completed for ${job.id}`);
   }
 }

packages/nocodb/src/modules/jobs/jobs/source-create/source-create.processor.ts

@@ -3,26 +3,40 @@ import { Process, Processor } from '@nestjs/bull';
 import { Job } from 'bull';
 import { JOBS_QUEUE, JobTypes } from '~/interface/Jobs';
 import { SourcesService } from '~/services/sources.service';
+import { JobsLogService } from '~/modules/jobs/jobs/jobs-log.service';
 
 @Processor(JOBS_QUEUE)
 export class SourceCreateProcessor {
-  private readonly debugLog = debug('nc:meta-sync:processor');
+  private readonly debugLog = debug('nc:jobs:source-create');
 
-  constructor(private readonly sourcesService: SourcesService) {}
+  constructor(
+    private readonly sourcesService: SourcesService,
+    private readonly jobsLogService: JobsLogService,
+  ) {}
 
   @Process(JobTypes.BaseCreate)
   async job(job: Job) {
+    this.debugLog(`job started for ${job.id}`);
+
     const { baseId, source } = job.data;
 
+    const logBasic = (log) => {
+      this.jobsLogService.sendLog(job, { message: log });
+      this.debugLog(log);
+    };
+
     const createdBase = await this.sourcesService.baseCreate({
       baseId,
       source,
+      logger: logBasic,
     });
 
     if (createdBase.isMeta()) {
       delete createdBase.config;
     }
 
+    this.debugLog(`job completed for ${job.id}`);
+
     return createdBase;
   }
 }

packages/nocodb/src/modules/jobs/jobs/source-delete/source-delete.processor.ts

@@ -6,18 +6,22 @@ import { SourcesService } from '~/services/sources.service';
 
 @Processor(JOBS_QUEUE)
 export class SourceDeleteProcessor {
-  private readonly debugLog = debug('nc:meta-sync:processor');
+  private readonly debugLog = debug('nc:jobs:source-delete');
 
   constructor(private readonly sourcesService: SourcesService) {}
 
   @Process(JobTypes.BaseDelete)
   async job(job: Job) {
+    this.debugLog(`job started for ${job.id}`);
+
     const { sourceId } = job.data;
 
     await this.sourcesService.baseDelete({
       sourceId,
     });
 
+    this.debugLog(`job completed for ${job.id}`);
+
     return true;
   }
 }

packages/nocodb/src/modules/jobs/redis/jobs-redis.service.ts

@@ -8,10 +8,7 @@ export class JobsRedisService {
   private unsubscribeCallbacks: { [key: string]: () => void } = {};
 
   constructor() {
-    if (process.env.NC_WORKER_CONTAINER === 'true') {
+    this.redisClient = new Redis(process.env.NC_REDIS_JOB_URL);
-      this.redisClient = new Redis(process.env.NC_REDIS_JOB_URL);
-      return;
-    }
     this.redisSubscriber = new Redis(process.env.NC_REDIS_JOB_URL);
   }
 

packages/nocodb/src/modules/jobs/redis/jobs.service.ts

@@ -11,13 +11,13 @@ export class JobsService implements OnModuleInit {
   // pause primary instance queue
   async onModuleInit() {
     if (process.env.NC_WORKER_CONTAINER !== 'true') {
-      await this.jobsQueue.pause(true);
+      // await this.jobsQueue.pause(true);
     }
   }
 
   async add(name: string, data: any) {
     // resume primary instance queue if there is no worker
-    const workerCount = (await this.jobsQueue.getWorkers()).length;
+    /* const workerCount = (await this.jobsQueue.getWorkers()).length;
     const localWorkerPaused = await this.jobsQueue.isPaused(true);
 
     // if there is no worker and primary instance queue is paused, resume it
@@ -26,7 +26,7 @@ export class JobsService implements OnModuleInit {
       await this.jobsQueue.resume(true);
     } else if (workerCount > 1 && !localWorkerPaused) {
       await this.jobsQueue.pause(true);
-    }
+    } */
 
     const job = await this.jobsQueue.add(name, data);
 

packages/nocodb/src/services/columns.service.ts

@@ -88,8 +88,8 @@ const nc_sanitizeName = (name) => {
 @Injectable()
 export class ColumnsService {
   constructor(
-    private readonly metaService: MetaService,
+    protected readonly metaService: MetaService,
-    private readonly appHooksService: AppHooksService,
+    protected readonly appHooksService: AppHooksService,
   ) {}
 
   async columnUpdate(param: {

packages/nocodb/src/services/filters.service.ts

@@ -8,7 +8,7 @@ import { Filter, Hook, View } from '~/models';
 
 @Injectable()
 export class FiltersService {
-  constructor(private readonly appHooksService: AppHooksService) {}
+  constructor(protected readonly appHooksService: AppHooksService) {}
 
   async hookFilterCreate(param: {
     filter: FilterReqType;

packages/nocodb/src/services/hooks.service.ts

@@ -13,7 +13,7 @@ import { Hook, HookLog, Model } from '~/models';
 
 @Injectable()
 export class HooksService {
-  constructor(private readonly appHooksService: AppHooksService) {}
+  constructor(protected readonly appHooksService: AppHooksService) {}
 
   validateHookPayload(notificationJsonOrObject: string | Record<string, any>) {
     let notification: { type?: string } = {};

packages/nocodb/src/services/sorts.service.ts

@@ -8,7 +8,7 @@ import { Sort } from '~/models';
 
 @Injectable()
 export class SortsService {
-  constructor(private appHooksService: AppHooksService) {}
+  constructor(protected readonly appHooksService: AppHooksService) {}
 
   async sortGet(param: { sortId: string }) {
     return Sort.get(param.sortId);

packages/nocodb/src/services/sources.service.ts

@@ -74,12 +74,19 @@ export class SourcesService {
     return true;
   }
 
-  async baseCreate(param: { baseId: string; source: BaseReqType }) {
+  async baseCreate(param: {
+    baseId: string;
+    source: BaseReqType;
+    logger?: (message: string) => void;
+  }) {
     validatePayload('swagger.json#/components/schemas/BaseReq', param.source);
 
     // type | base | baseId
     const baseBody = param.source;
     const base = await Base.getWithInfo(param.baseId);
+
+    param.logger?.('Creating the source');
+
     const source = await Source.createBase({
       ...baseBody,
       type: baseBody.config?.client,
@@ -88,7 +95,9 @@ export class SourcesService {
 
     await syncBaseMigration(base, source);
 
-    const info = await populateMeta(source, base);
+    param.logger?.('Populating meta');
+
+    const info = await populateMeta(source, base, param.logger);
 
     await populateRollupColumnAndHideLTAR(source, base);
 

packages/nocodb/src/services/tables.service.ts

@@ -33,9 +33,9 @@ import { validatePayload } from '~/helpers';
 @Injectable()
 export class TablesService {
   constructor(
-    private metaDiffService: MetaDiffsService,
+    protected readonly metaDiffService: MetaDiffsService,
-    private appHooksService: AppHooksService,
+    protected readonly appHooksService: AppHooksService,
-    private readonly columnsService: ColumnsService,
+    protected readonly columnsService: ColumnsService,
   ) {}
 
   async tableUpdate(param: {