Browse Source

feat: jobs socket

Signed-off-by: mertmit <mertmit99@gmail.com>
feat/export-nest
mertmit 2 years ago
parent
commit
542f0cad53
  1. 47
      packages/nc-gui/pages/index/index/index.vue
  2. 70
      packages/nc-gui/plugins/events.ts
  3. 1
      packages/nocodb-nest/src/models/Project.ts
  4. 5
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts
  5. 58
      packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts
  6. 71
      packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts
  7. 10
      packages/nocodb-nest/src/modules/jobs/jobs.module.ts
  8. 28
      packages/nocodb-nest/src/modules/jobs/jobs.service.ts
  9. 115
      packages/nocodb-nest/src/schema/swagger.json
  10. 2
      packages/nocodb-nest/src/services/projects.service.ts
  11. 84
      packages/nocodb-sdk/src/lib/Api.ts

47
packages/nc-gui/pages/index/index/index.vue

@ -27,7 +27,7 @@ definePageMeta({
title: 'title.myProject',
})
const { $api, $e } = useNuxtApp()
const { $api, $e, $events } = useNuxtApp()
const { api, isLoading } = useApi()
@ -74,6 +74,33 @@ const deleteProject = (project: ProjectType) => {
})
}
const duplicateProject = (project: ProjectType) => {
Modal.confirm({
title: `Do you want to duplicate '${project.title}' project?`,
wrapClassName: 'nc-modal-project-duplicate',
okText: 'Yes',
okType: 'primary',
cancelText: 'No',
async onOk() {
try {
const jobData = await api.project.duplicate(project.id as string)
$events.subscribe(jobData.type, jobData.id, async (data: { status: string }) => {
if (data.status === 'completed' || data.status === 'refresh') {
await loadProjects()
} else if (data.status === 'failed') {
message.error('Failed to duplicate project')
}
})
$e('a:project:duplicate')
} catch (e: any) {
message.error(await extractSdkResponseErrorMsg(e))
}
},
})
}
const handleProjectColor = async (projectId: string, color: string) => {
const tcolor = tinycolor(color)
@ -122,7 +149,7 @@ const getProjectPrimary = (project: ProjectType) => {
const customRow = (record: ProjectType) => ({
onClick: async () => {
await navigateTo(`/nc/${record.id}`)
if (record.status !== 'job') await navigateTo(`/nc/${record.id}`)
$e('a:project:open')
},
@ -249,8 +276,13 @@ const copyProjectMeta = async () => {
</a-menu>
</div>
<div
class="capitalize color-transition group-hover:text-primary !w-[400px] h-full overflow-hidden overflow-ellipsis whitespace-nowrap pl-2"
class="flex capitalize color-transition group-hover:text-primary !w-[400px] h-full overflow-hidden overflow-ellipsis whitespace-nowrap pl-2"
>
<component
:is="iconMap.reload"
v-if="record.status === 'job'"
:class="{ 'animate-infinite animate-spin text-gray-500': record.status === 'job' }"
/>
{{ text }}
</div>
</div>
@ -260,7 +292,7 @@ const copyProjectMeta = async () => {
<a-table-column key="id" :title="$t('labels.actions')" data-index="id">
<template #default="{ text, record }">
<div class="flex items-center gap-2">
<div v-if="record.status !== 'job'" class="flex items-center gap-2">
<component
:is="iconMap.edit"
v-e="['c:project:edit:rename']"
@ -268,6 +300,13 @@ const copyProjectMeta = async () => {
@click.stop="navigateTo(`/${text}`)"
/>
<component
:is="iconMap.copy"
v-e="['c:project:duplicate']"
class="nc-action-btn"
@click.stop="duplicateProject(record)"
/>
<component
:is="iconMap.delete"
class="nc-action-btn"

70
packages/nc-gui/plugins/events.ts

@ -0,0 +1,70 @@
import type { Socket } from 'socket.io-client'
import io from 'socket.io-client'
import { defineNuxtPlugin, useGlobal, watch } from '#imports'
export default defineNuxtPlugin(async (nuxtApp) => {
const { appInfo } = $(useGlobal())
let socket: Socket | null = null
const init = async (token: string) => {
try {
if (socket) socket.disconnect()
const url = new URL(appInfo.ncSiteUrl, window.location.href.split(/[?#]/)[0])
url.port = '8081'
socket = io(`${url.href}jobs`, {
extraHeaders: { 'xc-auth': token },
})
socket.on('connect_error', (e) => {
console.error(e)
socket?.disconnect()
})
} catch {}
}
if (nuxtApp.$state.signedIn.value) {
await init(nuxtApp.$state.token.value)
}
const events = {
subscribe(type: string, id: string, cb: (data: any) => void) {
if (socket) {
socket.emit('subscribe', { type, id })
const tempFn = (data: any) => {
if (data.id === id && data.type === type) {
cb(data)
if (data.status === 'completed' || data.status === 'failed') {
socket?.off('status', tempFn)
}
}
}
socket.on('status', tempFn)
}
},
getStatus(type: string, id: string): Promise<string> {
return new Promise((resolve) => {
if (socket) {
socket.emit('status', { type, id })
const tempFn = (data: any) => {
if (data.id === id && data.type === type) {
resolve(data.status)
socket?.off('status', tempFn)
}
}
socket.on('status', tempFn)
}
})
},
}
watch((nuxtApp.$state as ReturnType<typeof useGlobal>).token, (newToken, oldToken) => {
if (newToken && newToken !== oldToken) init(newToken)
else if (!newToken) socket?.disconnect()
})
nuxtApp.provide('events', events)
})

1
packages/nocodb-nest/src/models/Project.ts

@ -43,6 +43,7 @@ export default class Project implements ProjectType {
'prefix',
'description',
'is_meta',
'status',
]);
const { id: projectId } = await ncMeta.metaInsert2(

5
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.controller.ts

@ -22,7 +22,7 @@ export class DuplicateController {
@InjectQueue('duplicate') private readonly duplicateQueue: Queue,
) {}
@Post('/api/v1/db/meta/duplicate/:projectId/:baseId')
@Post('/api/v1/db/meta/duplicate/:projectId/:baseId?')
@HttpCode(200)
@Acl('duplicateBase')
async duplicateBase(
@ -30,7 +30,7 @@ export class DuplicateController {
@Param('projectId') projectId: string,
@Param('baseId') baseId?: string,
) {
await this.duplicateQueue.add('duplicate', {
const job = await this.duplicateQueue.add('duplicate', {
projectId,
baseId,
req: {
@ -38,5 +38,6 @@ export class DuplicateController {
clientIp: req.clientIp,
},
});
return { id: job.id, type: job.name };
}
}

58
packages/nocodb-nest/src/modules/jobs/export-import/duplicate.processor.ts

@ -17,24 +17,30 @@ import {
} from 'src/helpers/exportImportHelpers';
import { BulkDataAliasService } from 'src/services/bulk-data-alias.service';
import { UITypes } from 'nocodb-sdk';
import { JobsGateway } from '../jobs.gateway';
import { ExportService } from './export.service';
import { ImportService } from './import.service';
import type { LinkToAnotherRecordColumn } from 'src/models';
const DEBUG = false;
@Processor('duplicate')
export class DuplicateProcessor {
constructor(
private exportService: ExportService,
private importService: ImportService,
private projectsService: ProjectsService,
private bulkDataService: BulkDataAliasService,
private readonly exportService: ExportService,
private readonly importService: ImportService,
private readonly projectsService: ProjectsService,
private readonly bulkDataService: BulkDataAliasService,
private readonly jobsGateway: JobsGateway,
) {}
@OnQueueActive()
onActive(job: Job) {
console.log(
`Processing job ${job.id} of type ${job.name} with data ${job.data}...`,
);
this.jobsGateway.jobStatus({
type: job.name,
id: job.id.toString(),
status: 'active',
});
}
@OnQueueFailed()
@ -49,22 +55,37 @@ export class DuplicateProcessor {
},
),
);
this.jobsGateway.jobStatus({
type: job.name,
id: job.id.toString(),
status: 'failed',
});
}
@OnQueueCompleted()
onCompleted(job: Job) {
console.log(`Completed job ${job.id} of type ${job.name}!`);
this.jobsGateway.jobStatus({
type: job.name,
id: job.id.toString(),
status: 'completed',
});
}
@Process('duplicate')
async duplicateBase(job: Job) {
console.time('duplicateBase');
let start = process.hrtime();
const debugLog = function (...args: any[]) {
if (DEBUG) {
console.log(...args);
}
};
const elapsedTime = function (label?: string) {
const elapsedS = process.hrtime(start)[0].toFixed(3);
const elapsedMs = process.hrtime(start)[1] / 1000000;
if (label) console.log(`${label}: ${elapsedS}s ${elapsedMs}ms`);
if (label) debugLog(`${label}: ${elapsedS}s ${elapsedMs}ms`);
start = process.hrtime();
};
@ -108,10 +129,16 @@ export class DuplicateProcessor {
);
const dupProject = await this.projectsService.projectCreate({
project: { title: uniqueTitle },
project: { title: uniqueTitle, status: 'job' },
user: { id: user.id },
});
this.jobsGateway.jobStatus({
type: job.name,
id: job.id.toString(),
status: 'refresh',
});
const dupBaseId = dupProject.bases[0].id;
elapsedTime('projectCreate');
@ -177,7 +204,7 @@ export class DuplicateProcessor {
headers.push(col.column_name);
}
} else {
console.log('header not found', header);
debugLog('header not found', header);
}
}
parser.resume();
@ -331,6 +358,11 @@ export class DuplicateProcessor {
}
elapsedTime('links');
console.timeEnd('duplicateBase');
await this.projectsService.projectUpdate({
projectId: dupProject.id,
project: {
status: null,
},
});
}
}

71
packages/nocodb-nest/src/modules/jobs/jobs.gateway.ts

@ -0,0 +1,71 @@
import {
ConnectedSocket,
MessageBody,
SubscribeMessage,
WebSocketGateway,
WebSocketServer,
} from '@nestjs/websockets';
import { Server, Socket } from 'socket.io';
import { Injectable, UseGuards } from '@nestjs/common';
import { GlobalGuard } from 'src/guards/global/global.guard';
import { ExtractProjectIdMiddleware } from 'src/middlewares/extract-project-id/extract-project-id.middleware';
import { JobsService } from './jobs.service';
@WebSocketGateway(8081, {
cors: {
origin: '*',
},
allowedHeaders: ['xc-auth'],
credentials: true,
namespace: 'jobs',
})
@Injectable()
export class JobsGateway {
constructor(private readonly jobsService: JobsService) {}
@WebSocketServer()
server: Server;
@SubscribeMessage('subscribe')
async subscribe(
@MessageBody() data: { type: string; id: string },
@ConnectedSocket() client: Socket,
): Promise<void> {
const rooms = await this.jobsService.jobList(data.type);
const room = rooms.find((r) => r.id === data.id);
if (room) {
client.join(data.id);
}
}
@SubscribeMessage('status')
async status(
@MessageBody() data: { type: string; id: string },
@ConnectedSocket() client: Socket,
): Promise<void> {
client.emit('status', {
id: data.id,
type: data.type,
status: await this.jobsService.jobStatus(data.type, data.id),
});
}
async jobStatus(data: {
type: string;
id: string;
status:
| 'completed'
| 'waiting'
| 'active'
| 'delayed'
| 'failed'
| 'paused'
| 'refresh';
}): Promise<void> {
this.server.to(data.id).emit('status', {
id: data.id,
type: data.type,
status: data.status,
});
}
}

10
packages/nocodb-nest/src/modules/jobs/jobs.module.ts

@ -3,10 +3,12 @@ import { BullModule } from '@nestjs/bull';
import { GlobalModule } from '../global/global.module';
import { DatasModule } from '../datas/datas.module';
import { MetasModule } from '../metas/metas.module';
import { JobsService } from './jobs.service';
import { ExportService } from './export-import/export.service';
import { ImportService } from './export-import/import.service';
import { DuplicateController } from './export-import/duplicate.controller';
import { DuplicateProcessor } from './export-import/duplicate.processor';
import { JobsGateway } from './jobs.gateway';
@Module({
imports: [
@ -18,6 +20,12 @@ import { DuplicateProcessor } from './export-import/duplicate.processor';
}),
],
controllers: [DuplicateController],
providers: [DuplicateProcessor, ExportService, ImportService],
providers: [
JobsGateway,
JobsService,
DuplicateProcessor,
ExportService,
ImportService,
],
})
export class JobsModule {}

28
packages/nocodb-nest/src/modules/jobs/jobs.service.ts

@ -0,0 +1,28 @@
import { InjectQueue } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { Queue } from 'bull';
@Injectable()
export class JobsService {
constructor(@InjectQueue('duplicate') private duplicateQueue: Queue) {}
async jobStatus(jobType: string, jobId: string) {
switch (jobType) {
case 'duplicate':
default:
return await (await this.duplicateQueue.getJob(jobId)).getState();
}
}
async jobList(jobType: string) {
switch (jobType) {
case 'duplicate':
default:
return await this.duplicateQueue.getJobs([
'active',
'waiting',
'delayed',
]);
}
}
}

115
packages/nocodb-nest/src/schema/swagger.json

@ -2100,6 +2100,111 @@
]
}
},
"/api/v1/db/meta/duplicate/{projectId}/{baseId}": {
"post": {
"summary": "Duplicate Project Base",
"operationId": "project-base-duplicate",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"type": {
"type": "string"
},
"id": {
"type": "string"
}
}
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest"
}
},
"tags": ["Project"],
"description": "Duplicate a project",
"parameters": [
{
"$ref": "#/components/parameters/xc-auth"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "p_124hhlkbeasewh",
"type": "string"
},
"name": "projectId",
"in": "path",
"required": true,
"description": "Unique Project ID"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "ds_124hhlkbeasewh",
"type": "string"
},
"name": "baseId",
"in": "path",
"required": false,
"description": "Unique Base ID"
}
]
}
},
"/api/v1/db/meta/duplicate/{projectId}": {
"post": {
"summary": "Duplicate Project",
"operationId": "project-duplicate",
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"type": {
"type": "string"
},
"id": {
"type": "string"
}
}
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest"
}
},
"tags": ["Project"],
"description": "Duplicate a project",
"parameters": [
{
"$ref": "#/components/parameters/xc-auth"
},
{
"schema": {
"$ref": "#/components/schemas/Id",
"example": "p_124hhlkbeasewh",
"type": "string"
},
"name": "projectId",
"in": "path",
"required": true,
"description": "Unique Project ID"
}
]
}
},
"/api/v1/db/meta/projects/{projectId}": {
"parameters": [
{
@ -18256,6 +18361,11 @@
"maxLength": 128,
"minLength": 1,
"type": "string"
},
"status": {
"$ref": "#/components/schemas/StringOrNull",
"description": "Project Status",
"example": "locked"
}
},
"required": ["title"],
@ -18299,6 +18409,11 @@
"maxLength": 128,
"minLength": 1,
"type": "string"
},
"status": {
"$ref": "#/components/schemas/StringOrNull",
"description": "Project Status",
"example": "locked"
}
}
},

2
packages/nocodb-nest/src/services/projects.service.ts

@ -58,7 +58,7 @@ export class ProjectsService {
const data: Partial<Project> = extractPropsAndSanitize(
param?.project as Project,
['title', 'meta', 'color'],
['title', 'meta', 'color', 'status'],
);
if (

84
packages/nocodb-sdk/src/lib/Api.ts

@ -1892,6 +1892,11 @@ export interface ProjectReqType {
* @example My Project
*/
title: string;
/**
* Project Status
* @example locked
*/
status?: StringOrNullType;
}
/**
@ -1910,6 +1915,11 @@ export interface ProjectUpdateReqType {
* @example My Project
*/
title?: string;
/**
* Project Status
* @example locked
*/
status?: StringOrNullType;
}
/**
@ -4001,6 +4011,80 @@ export class Api<
...params,
}),
/**
* @description Duplicate a project
*
* @tags Project
* @name BaseDuplicate
* @summary Duplicate Project Base
* @request POST:/api/v1/db/meta/duplicate/{projectId}/{baseId}
* @response `200` `{
type?: string,
id?: string,
}` OK
* @response `400` `{
\** @example BadRequest [Error]: <ERROR MESSAGE> *\
msg: string,
}`
*/
baseDuplicate: (
projectId: IdType,
baseId?: IdType,
params: RequestParams = {}
) =>
this.request<
{
type?: string;
id?: string;
},
{
/** @example BadRequest [Error]: <ERROR MESSAGE> */
msg: string;
}
>({
path: `/api/v1/db/meta/duplicate/${projectId}/${baseId}`,
method: 'POST',
format: 'json',
...params,
}),
/**
* @description Duplicate a project
*
* @tags Project
* @name Duplicate
* @summary Duplicate Project
* @request POST:/api/v1/db/meta/duplicate/{projectId}
* @response `200` `{
type?: string,
id?: string,
}` OK
* @response `400` `{
\** @example BadRequest [Error]: <ERROR MESSAGE> *\
msg: string,
}`
*/
duplicate: (projectId: IdType, params: RequestParams = {}) =>
this.request<
{
type?: string;
id?: string;
},
{
/** @example BadRequest [Error]: <ERROR MESSAGE> */
msg: string;
}
>({
path: `/api/v1/db/meta/duplicate/${projectId}`,
method: 'POST',
format: 'json',
...params,
}),
/**
* @description Get the info of a given project
*

Loading…
Cancel
Save