feat: jobs socket

Signed-off-by: mertmit <mertmit99@gmail.com>

packages/nc-gui/pages/index/index/index.vue

@@ -27,7 +27,7 @@ definePageMeta({
   title: 'title.myProject',
 })
 
-const { $api, $e } = useNuxtApp()
+const { $api, $e, $events } = useNuxtApp()
 
 const { api, isLoading } = useApi()
 
@@ -74,6 +74,33 @@ const deleteProject = (project: ProjectType) => {
   })
 }
 
+const duplicateProject = (project: ProjectType) => {
+  Modal.confirm({
+    title: `Do you want to duplicate '${project.title}' project?`,
+    wrapClassName: 'nc-modal-project-duplicate',
+    okText: 'Yes',
+    okType: 'primary',
+    cancelText: 'No',
+    async onOk() {
+      try {
+        const jobData = await api.project.duplicate(project.id as string)
+
+        $events.subscribe(jobData.type, jobData.id, async (data: { status: string }) => {
+          if (data.status === 'completed' || data.status === 'refresh') {
+            await loadProjects()
+          } else if (data.status === 'failed') {
+            message.error('Failed to duplicate project')
+          }
+        })
+
+        $e('a:project:duplicate')
+      } catch (e: any) {
+        message.error(await extractSdkResponseErrorMsg(e))
+      }
+    },
+  })
+}
+
 const handleProjectColor = async (projectId: string, color: string) => {
   const tcolor = tinycolor(color)
 
@@ -122,7 +149,7 @@ const getProjectPrimary = (project: ProjectType) => {
 
 const customRow = (record: ProjectType) => ({
   onClick: async () => {
-    await navigateTo(`/nc/${record.id}`)
+    if (record.status !== 'job') await navigateTo(`/nc/${record.id}`)
 
     $e('a:project:open')
   },
@@ -249,8 +276,13 @@ const copyProjectMeta = async () => {
               </a-menu>
             </div>
             <div
-              class="capitalize color-transition group-hover:text-primary !w-[400px] h-full overflow-hidden overflow-ellipsis whitespace-nowrap pl-2"
+              class="flex capitalize color-transition group-hover:text-primary !w-[400px] h-full overflow-hidden overflow-ellipsis whitespace-nowrap pl-2"
             >
+              <component
+                :is="iconMap.reload"
+                v-if="record.status === 'job'"
+                :class="{ 'animate-infinite animate-spin text-gray-500': record.status === 'job' }"
+              />
               {{ text }}
             </div>
           </div>
@@ -260,7 +292,7 @@ const copyProjectMeta = async () => {
 
       <a-table-column key="id" :title="$t('labels.actions')" data-index="id">
         <template #default="{ text, record }">
-          <div class="flex items-center gap-2">
+          <div v-if="record.status !== 'job'" class="flex items-center gap-2">
             <component
               :is="iconMap.edit"
               v-e="['c:project:edit:rename']"
@@ -268,6 +300,13 @@ const copyProjectMeta = async () => {
               @click.stop="navigateTo(`/${text}`)"
             />
 
+            <component
+              :is="iconMap.copy"
+              v-e="['c:project:duplicate']"
+              class="nc-action-btn"
+              @click.stop="duplicateProject(record)"
+            />
+
             <component
               :is="iconMap.delete"
               class="nc-action-btn"

packages/nc-gui/plugins/events.ts

@@ -0,0 +1,70 @@
+import type { Socket } from 'socket.io-client'
+import io from 'socket.io-client'
+import { defineNuxtPlugin, useGlobal, watch } from '#imports'
+
+export default defineNuxtPlugin(async (nuxtApp) => {
+  const { appInfo } = $(useGlobal())
+
+  let socket: Socket | null = null
+
+  const init = async (token: string) => {
+    try {
+      if (socket) socket.disconnect()
+
+      const url = new URL(appInfo.ncSiteUrl, window.location.href.split(/[?#]/)[0])
+
+      url.port = '8081'
+
+      socket = io(`${url.href}jobs`, {
+        extraHeaders: { 'xc-auth': token },
+      })
+
+      socket.on('connect_error', (e) => {
+        console.error(e)
+        socket?.disconnect()
+      })
+    } catch {}
+  }
+
+  if (nuxtApp.$state.signedIn.value) {
+    await init(nuxtApp.$state.token.value)
+  }
+
+  const events = {
+    subscribe(type: string, id: string, cb: (data: any) => void) {
+      if (socket) {
+        socket.emit('subscribe', { type, id })
+        const tempFn = (data: any) => {
+          if (data.id === id && data.type === type) {
+            cb(data)
+            if (data.status === 'completed' || data.status === 'failed') {
+              socket?.off('status', tempFn)
+            }
+          }
+        }
+        socket.on('status', tempFn)
+      }
+    },
+    getStatus(type: string, id: string): Promise<string> {
+      return new Promise((resolve) => {
+        if (socket) {
+          socket.emit('status', { type, id })
+          const tempFn = (data: any) => {
+            if (data.id === id && data.type === type) {
+              resolve(data.status)
+              socket?.off('status', tempFn)
+            }
+          }
+          socket.on('status', tempFn)
+        }
+      })
+    },
+  }
+
+  watch((nuxtApp.$state as ReturnType<typeof useGlobal>).token, (newToken, oldToken) => {
+    if (newToken && newToken !== oldToken) init(newToken)
+    else if (!newToken) socket?.disconnect()
+  })
+
+  nuxtApp.provide('events', events)
+})

packages/nocodb-nest/src/models/Project.ts

@@ -43,6 +43,7 @@ export default class Project implements ProjectType {
       'prefix',
       'description',
       'is_meta',
+      'status',
     ]);
 
     const { id: projectId } = await ncMeta.metaInsert2(

packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts

@@ -22,7 +22,7 @@ export class DuplicateController {
     @InjectQueue('duplicate') private readonly duplicateQueue: Queue,
   ) {}
 
-  @Post('/api/v1/db/meta/duplicate/:projectId/:baseId')
+  @Post('/api/v1/db/meta/duplicate/:projectId/:baseId?')
   @HttpCode(200)
   @Acl('duplicateBase')
   async duplicateBase(
@@ -30,7 +30,7 @@ export class DuplicateController {
     @Param('projectId') projectId: string,
     @Param('baseId') baseId?: string,
   ) {
-    await this.duplicateQueue.add('duplicate', {
+    const job = await this.duplicateQueue.add('duplicate', {
       projectId,
       baseId,
       req: {
@@ -38,5 +38,6 @@ export class DuplicateController {
         clientIp: req.clientIp,
       },
     });
+    return { id: job.id, type: job.name };
   }
 }

packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts

@@ -17,24 +17,30 @@ import {
 } from 'src/helpers/exportImportHelpers';
 import { BulkDataAliasService } from 'src/services/bulk-data-alias.service';
 import { UITypes } from 'nocodb-sdk';
+import { JobsGateway } from '../jobs.gateway';
 import { ExportService } from './export.service';
 import { ImportService } from './import.service';
 import type { LinkToAnotherRecordColumn } from 'src/models';
 
+const DEBUG = false;
+
 @Processor('duplicate')
 export class DuplicateProcessor {
   constructor(
-    private exportService: ExportService,
+    private readonly exportService: ExportService,
-    private importService: ImportService,
+    private readonly importService: ImportService,
-    private projectsService: ProjectsService,
+    private readonly projectsService: ProjectsService,
-    private bulkDataService: BulkDataAliasService,
+    private readonly bulkDataService: BulkDataAliasService,
+    private readonly jobsGateway: JobsGateway,
   ) {}
 
   @OnQueueActive()
   onActive(job: Job) {
-    console.log(
+    this.jobsGateway.jobStatus({
-      `Processing job ${job.id} of type ${job.name} with data ${job.data}...`,
+      type: job.name,
-    );
+      id: job.id.toString(),
+      status: 'active',
+    });
   }
 
   @OnQueueFailed()
@@ -49,22 +55,37 @@ export class DuplicateProcessor {
         },
       ),
     );
+
+    this.jobsGateway.jobStatus({
+      type: job.name,
+      id: job.id.toString(),
+      status: 'failed',
+    });
   }
 
   @OnQueueCompleted()
   onCompleted(job: Job) {
-    console.log(`Completed job ${job.id} of type ${job.name}!`);
+    this.jobsGateway.jobStatus({
+      type: job.name,
+      id: job.id.toString(),
+      status: 'completed',
+    });
   }
 
   @Process('duplicate')
   async duplicateBase(job: Job) {
-    console.time('duplicateBase');
     let start = process.hrtime();
 
+    const debugLog = function (...args: any[]) {
+      if (DEBUG) {
+        console.log(...args);
+      }
+    };
+
     const elapsedTime = function (label?: string) {
       const elapsedS = process.hrtime(start)[0].toFixed(3);
       const elapsedMs = process.hrtime(start)[1] / 1000000;
-      if (label) console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`);
+      if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
       start = process.hrtime();
     };
 
@@ -108,10 +129,16 @@ export class DuplicateProcessor {
     );
 
     const dupProject = await this.projectsService.projectCreate({
-      project: { title: uniqueTitle },
+      project: { title: uniqueTitle, status: 'job' },
       user: { id: user.id },
     });
 
+    this.jobsGateway.jobStatus({
+      type: job.name,
+      id: job.id.toString(),
+      status: 'refresh',
+    });
+
     const dupBaseId = dupProject.bases[0].id;
 
     elapsedTime('projectCreate');
@@ -177,7 +204,7 @@ export class DuplicateProcessor {
                     headers.push(col.column_name);
                   }
                 } else {
-                  console.log('header not found', header);
+                  debugLog('header not found', header);
                 }
               }
               parser.resume();
@@ -331,6 +358,11 @@ export class DuplicateProcessor {
     }
 
     elapsedTime('links');
-    console.timeEnd('duplicateBase');
+    await this.projectsService.projectUpdate({
+      projectId: dupProject.id,
+      project: {
+        status: null,
+      },
+    });
   }
 }

packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts

@@ -0,0 +1,71 @@
+import {
+  ConnectedSocket,
+  MessageBody,
+  SubscribeMessage,
+  WebSocketGateway,
+  WebSocketServer,
+} from '@nestjs/websockets';
+import { Server, Socket } from 'socket.io';
+import { Injectable, UseGuards } from '@nestjs/common';
+import { GlobalGuard } from 'src/guards/global/global.guard';
+import { ExtractProjectIdMiddleware } from 'src/middlewares/extract-project-id/extract-project-id.middleware';
+import { JobsService } from './jobs.service';
+
+@WebSocketGateway(8081, {
+  cors: {
+    origin: '*',
+  },
+  allowedHeaders: ['xc-auth'],
+  credentials: true,
+  namespace: 'jobs',
+})
+@Injectable()
+export class JobsGateway {
+  constructor(private readonly jobsService: JobsService) {}
+
+  @WebSocketServer()
+  server: Server;
+
+  @SubscribeMessage('subscribe')
+  async subscribe(
+    @MessageBody() data: { type: string; id: string },
+    @ConnectedSocket() client: Socket,
+  ): Promise<void> {
+    const rooms = await this.jobsService.jobList(data.type);
+    const room = rooms.find((r) => r.id === data.id);
+    if (room) {
+      client.join(data.id);
+    }
+  }
+
+  @SubscribeMessage('status')
+  async status(
+    @MessageBody() data: { type: string; id: string },
+    @ConnectedSocket() client: Socket,
+  ): Promise<void> {
+    client.emit('status', {
+      id: data.id,
+      type: data.type,
+      status: await this.jobsService.jobStatus(data.type, data.id),
+    });
+  }
+
+  async jobStatus(data: {
+    type: string;
+    id: string;
+    status:
+      | 'completed'
+      | 'waiting'
+      | 'active'
+      | 'delayed'
+      | 'failed'
+      | 'paused'
+      | 'refresh';
+  }): Promise<void> {
+    this.server.to(data.id).emit('status', {
+      id: data.id,
+      type: data.type,
+      status: data.status,
+    });
+  }
+}

packages/nocodb-nest/src/modules/jobs/jobs.module.ts

@@ -3,10 +3,12 @@ import { BullModule } from '@nestjs/bull';
 import { GlobalModule } from '../global/global.module';
 import { DatasModule } from '../datas/datas.module';
 import { MetasModule } from '../metas/metas.module';
+import { JobsService } from './jobs.service';
 import { ExportService } from './export-import/export.service';
 import { ImportService } from './export-import/import.service';
 import { DuplicateController } from './export-import/duplicate.controller';
 import { DuplicateProcessor } from './export-import/duplicate.processor';
+import { JobsGateway } from './jobs.gateway';
 
 @Module({
   imports: [
@@ -18,6 +20,12 @@ import { DuplicateProcessor } from './export-import/duplicate.processor';
     }),
   ],
   controllers: [DuplicateController],
-  providers: [DuplicateProcessor, ExportService, ImportService],
+  providers: [
+    JobsGateway,
+    JobsService,
+    DuplicateProcessor,
+    ExportService,
+    ImportService,
+  ],
 })
 export class JobsModule {}

packages/nocodb-nest/src/modules/jobs/jobs.service.ts

@@ -0,0 +1,28 @@
+import { InjectQueue } from '@nestjs/bull';
+import { Injectable } from '@nestjs/common';
+import { Queue } from 'bull';
+
+@Injectable()
+export class JobsService {
+  constructor(@InjectQueue('duplicate') private duplicateQueue: Queue) {}
+
+  async jobStatus(jobType: string, jobId: string) {
+    switch (jobType) {
+      case 'duplicate':
+      default:
+        return await (await this.duplicateQueue.getJob(jobId)).getState();
+    }
+  }
+
+  async jobList(jobType: string) {
+    switch (jobType) {
+      case 'duplicate':
+      default:
+        return await this.duplicateQueue.getJobs([
+          'active',
+          'waiting',
+          'delayed',
+        ]);
+    }
+  }
+}

packages/nocodb-nest/src/schema/swagger.json

@@ -2100,6 +2100,111 @@
         ]
       }
     },
+    "/api/v1/db/meta/duplicate/{projectId}/{baseId}": {
+      "post": {
+        "summary": "Duplicate Project Base",
+        "operationId": "project-base-duplicate",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "type": {
+                      "type": "string"
+                    },
+                    "id": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "$ref": "#/components/responses/BadRequest"
+          }
+        },
+        "tags": ["Project"],
+        "description": "Duplicate a project",
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/xc-auth"
+          },
+          {
+            "schema": {
+              "$ref": "#/components/schemas/Id",
+              "example": "p_124hhlkbeasewh",
+              "type": "string"
+            },
+            "name": "projectId",
+            "in": "path",
+            "required": true,
+            "description": "Unique Project ID"
+          },
+          {
+            "schema": {
+              "$ref": "#/components/schemas/Id",
+              "example": "ds_124hhlkbeasewh",
+              "type": "string"
+            },
+            "name": "baseId",
+            "in": "path",
+            "required": false,
+            "description": "Unique Base ID"
+          }
+        ]
+      }
+    },
+    "/api/v1/db/meta/duplicate/{projectId}": {
+      "post": {
+        "summary": "Duplicate Project",
+        "operationId": "project-duplicate",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "type": {
+                      "type": "string"
+                    },
+                    "id": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          },
+          "400": {
+            "$ref": "#/components/responses/BadRequest"
+          }
+        },
+        "tags": ["Project"],
+        "description": "Duplicate a project",
+        "parameters": [
+          {
+            "$ref": "#/components/parameters/xc-auth"
+          },
+          {
+            "schema": {
+              "$ref": "#/components/schemas/Id",
+              "example": "p_124hhlkbeasewh",
+              "type": "string"
+            },
+            "name": "projectId",
+            "in": "path",
+            "required": true,
+            "description": "Unique Project ID"
+          }
+        ]
+      }
+    },
     "/api/v1/db/meta/projects/{projectId}": {
       "parameters": [
         {
@@ -18256,6 +18361,11 @@
             "maxLength": 128,
             "minLength": 1,
             "type": "string"
+          },
+          "status": {
+            "$ref": "#/components/schemas/StringOrNull",
+            "description": "Project Status",
+            "example": "locked"
           }
         },
         "required": ["title"],
@@ -18299,6 +18409,11 @@
             "maxLength": 128,
             "minLength": 1,
             "type": "string"
+          },
+          "status": {
+            "$ref": "#/components/schemas/StringOrNull",
+            "description": "Project Status",
+            "example": "locked"
           }
         }
       },

packages/nocodb-nest/src/services/projects.service.ts

@@ -58,7 +58,7 @@ export class ProjectsService {
 
     const data: Partial<Project> = extractPropsAndSanitize(
       param?.project as Project,
-      ['title', 'meta', 'color'],
+      ['title', 'meta', 'color', 'status'],
     );
 
     if (

packages/nocodb-sdk/src/lib/Api.ts

@@ -1892,6 +1892,11 @@ export interface ProjectReqType {
    * @example My Project
    */
   title: string;
+  /**
+   * Project Status
+   * @example locked
+   */
+  status?: StringOrNullType;
 }
 
 /**
@@ -1910,6 +1915,11 @@ export interface ProjectUpdateReqType {
    * @example My Project
    */
   title?: string;
+  /**
+   * Project Status
+   * @example locked
+   */
+  status?: StringOrNullType;
 }
 
 /**
@@ -4001,6 +4011,80 @@ export class Api<
         ...params,
       }),
 
+    /**
+ * @description Duplicate a project
+ * 
+ * @tags Project
+ * @name BaseDuplicate
+ * @summary Duplicate Project Base
+ * @request POST:/api/v1/db/meta/duplicate/{projectId}/{baseId}
+ * @response `200` `{
+  type?: string,
+  id?: string,
+
+}` OK
+ * @response `400` `{
+  \** @example BadRequest [Error]: <ERROR MESSAGE> *\
+  msg: string,
+
+}`
+ */
+    baseDuplicate: (
+      projectId: IdType,
+      baseId?: IdType,
+      params: RequestParams = {}
+    ) =>
+      this.request<
+        {
+          type?: string;
+          id?: string;
+        },
+        {
+          /** @example BadRequest [Error]: <ERROR MESSAGE> */
+          msg: string;
+        }
+      >({
+        path: `/api/v1/db/meta/duplicate/${projectId}/${baseId}`,
+        method: 'POST',
+        format: 'json',
+        ...params,
+      }),
+
+    /**
+ * @description Duplicate a project
+ * 
+ * @tags Project
+ * @name Duplicate
+ * @summary Duplicate Project
+ * @request POST:/api/v1/db/meta/duplicate/{projectId}
+ * @response `200` `{
+  type?: string,
+  id?: string,
+
+}` OK
+ * @response `400` `{
+  \** @example BadRequest [Error]: <ERROR MESSAGE> *\
+  msg: string,
+
+}`
+ */
+    duplicate: (projectId: IdType, params: RequestParams = {}) =>
+      this.request<
+        {
+          type?: string;
+          id?: string;
+        },
+        {
+          /** @example BadRequest [Error]: <ERROR MESSAGE> */
+          msg: string;
+        }
+      >({
+        path: `/api/v1/db/meta/duplicate/${projectId}`,
+        method: 'POST',
+        format: 'json',
+        ...params,
+      }),
+
     /**
  * @description Get the info of a given project
  *