Browse Source

fix: airtable import (WIP)

Signed-off-by: Pranav C <pranavxc@gmail.com>
pull/5444/head
Pranav C 1 year ago
parent
commit
9f2f75fd95
  1. 208
      packages/nocodb-nest/src/Noco.ts
  2. 2
      packages/nocodb-nest/src/app.module.ts
  3. 7
      packages/nocodb-nest/src/modules/global/global.module.ts
  4. 24
      packages/nocodb-nest/src/modules/import/import.controller.ts
  5. 4
      packages/nocodb-nest/src/modules/import/import.module.ts
  6. 10
      packages/nocodb-nest/src/modules/sync/helpers/job.ts
  7. 36
      packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts
  8. 13
      packages/nocodb-nest/src/run/docker.ts
  9. 6
      packages/nocodb-nest/src/run/dockerRunMysql.ts
  10. 6
      packages/nocodb-nest/src/run/dockerRunPG.ts
  11. 5
      packages/nocodb-nest/src/run/dockerRunPG_CyQuick.ts
  12. 8
      packages/nocodb-nest/src/run/testDocker.ts
  13. 83
      packages/nocodb-nest/src/services/client/client.service.ts
  14. 8
      packages/nocodb-nest/src/services/client/socket.service.spec.ts
  15. 81
      packages/nocodb-nest/src/services/client/socket.service.ts

208
packages/nocodb-nest/src/Noco.ts

@ -1,14 +1,21 @@
// import * as Sentry from '@sentry/node';
import { NestFactory } from '@nestjs/core';
import clear from 'clear';
import * as express from 'express';
import NcToolGui from 'nc-lib-gui';
import { AppModule } from './app.module';
import { NC_LICENSE_KEY } from './constants';
import Store from './models/Store';
import type { Express } from 'express';
import type * as http from 'http';
export default class Noco {
private static _this: Noco;
private static ee: boolean;
public static readonly env: string = '_noco';
private static _httpServer: http.Server;
private static _server: Express;
public static get dashboardUrl(): string {
let siteUrl = `http://localhost:${process.env.PORT || 8080}`;
@ -21,24 +28,6 @@ export default class Noco {
return `${siteUrl}${Noco._this?.config?.dashboardPath}`;
}
/*
public static async init(
args?: {
progressCallback?: Function;
registerRoutes?: Function;
registerGql?: Function;
registerContext?: Function;
afterMetaMigrationInit?: Function;
},
server?: http.Server,
app?: express.Express
): Promise<Router> {
if (Noco._this) {
return Noco._this.router;
}
Noco._this = new Noco();
return Noco._this.init(args, server, app);
}*/
public static config: any;
public readonly router: express.Router;
@ -65,172 +54,10 @@ export default class Noco {
this.router = express.Router();
this.projectRouter = express.Router();
/******************* prints : start *******************/
// this.sumTable = new Table({
// head: ['#DBs', '#Tables',
// '#GQL\nServers', '#REST\nServers',
// '#APIs',
// 'Time\ntaken',
// // 'If avg manual effort\nper api = 15 minutes\nand\nAPI developer salary = $76k'
// ].map(v => colors.green(v))
// , colWidths: [10, 12, 9, 9, 12, 12]
// });
// this.table = new Table({
// colWidths: [4, 8, 8, 20, 9, 7, 35, 9],
// head: ['#', 'DB\nType', 'API\nType', 'Database', '#Tables', '#APIs', 'APIs URL', 'Time\ntaken'].map(v => colors.green(v))
// });
clear();
/******************* prints : end *******************/
}
/* public async init(
args?: {
progressCallback?: Function;
registerRoutes?: Function;
registerGql?: Function;
registerContext?: Function;
afterMetaMigrationInit?: Function;
},
server?: http.Server,
_app?: express.Express,
) {
/!* prepare config *!/
Noco.config = this.config = await NcConfigFactory.make();
/!******************* setup : start *******************!/
this.env = '_noco'; //process.env['NODE_ENV'] || this.config.workingEnv || 'dev';
this.config.workingEnv = this.env;
this.config.type = 'docker';
if (!this.config.toolDir) {
this.config.toolDir = process.cwd();
}
// this.ncToolApi = new NcToolGui(this.config);
// if (server) {
// server.set('view engine', 'ejs');
// }
const NcMetaImpl = process.env.EE ? NcMetaImplEE : NcMetaImplCE;
// const NcMetaMgr = process.env.EE ? NcMetaMgrEE : NcMetaMgrCE;
Noco._ncMeta = new NcMetaImpl(this, this.config);
// this.metaMgr = new NcMetaMgr(this, this.config, Noco._ncMeta);
// this.metaMgrv2 = new NcMetaMgrv2(this, this.config, Noco._ncMeta);
/!******************* setup : end *******************!/
// @ts-ignore
const {
progressCallback,
// registerRoutes,
// registerContext,
// registerGql
} = args || {};
log('Initializing app');
// create tool directory if missing
await mkdirp(this.config.toolDir);
this.initSentry();
NocoCache.init();
// this.apiInfInfoList = [];
//
// this.startTime = Date.now();
if (!this.config.try) {
await NcConfigFactory.metaDbCreateIfNotExist(this.config);
await this.syncMigration();
}
await Noco._ncMeta.metaInit();
await Noco.loadEEState();
await this.initJwt();
if (args?.afterMetaMigrationInit) {
await args.afterMetaMigrationInit();
}
/!******************* Middlewares : start *******************!/
this.router.use((req: any, _res, next) => {
req.nc = this.requestContext;
req.ncSiteUrl =
this.config?.envs?.[this.env]?.publicUrl ||
this.config?.publicUrl ||
req.protocol + '://' + req.get('host');
req.ncFullUrl = req.protocol + '://' + req.get('host') + req.originalUrl;
next();
});
// to get ip addresses
this.router.use(requestIp.mw());
this.router.use(cookieParser());
this.router.use(
bodyParser.json({
limit: process.env.NC_REQUEST_BODY_SIZE || '50mb',
}),
);
this.router.use(morgan('tiny'));
this.router.use(express.static(path.join(__dirname, './public')));
this.router.use((req: any, _res, next) => {
req.ncProjectId = req?.query?.project_id || req?.body?.project_id;
next();
});
/!* this.router.use(this.config.dashboardPath, (req: any, _res, next) => {
req.ncProjectId = req?.body?.project_id;
next();
})*!/
this.router.use(`/nc/:project_id/!*`, (req: any, _res, next) => {
req.ncProjectId = req.ncProjectId || req.params.project_id;
next();
});
this.router.use(MetaAPILogger.mw);
/!******************* Middlewares : end *******************!/
// await this.initProjectBuilders();
// const runTimeHandler = this.handleRuntimeChanges(progressCallback);
// this.ncToolApi.addListener(runTimeHandler);
// this.metaMgr.setListener(runTimeHandler);
// this.metaMgrv2.setListener(runTimeHandler);
// await this.metaMgr.initHandler(this.router);
// await this.metaMgrv2.initHandler(this.router);
await NcPluginMgrv2.init(Noco.ncMeta);
registerMetaApis(this.router, server);
// this.router.use(
// this.config.dashboardPath,
// await this.ncToolApi.expressMiddleware()
// );
this.router.use(NcToolGui.expressMiddleware(this.config.dashboardPath));
this.router.get('/', (_req, res) =>
res.redirect(this.config.dashboardPath),
);
this.initSentryErrorHandler();
/!* catch error *!/
this.router.use((err, _req, res, next) => {
if (err) {
return res.status(400).json({ msg: err.message });
}
next();
});
T.init({
instance: getInstance,
});
T.emit('evt_app_started', await User.count());
console.log(`App started successfully.\nVisit -> ${Noco.dashboardUrl}`);
weAreHiring();
return this.router;
}*/
public getConfig(): any {
return this.config;
}
@ -265,4 +92,25 @@ export default class Noco {
} catch {}
return (Noco.ee = false);
}
static async init(param: any, httpServer: http.Server, server: Express) {
this._httpServer = httpServer;
this._server = server;
const nestApp = await NestFactory.create(AppModule);
await nestApp.init();
const dashboardPath = process.env.NC_DASHBOARD_URL || '/dashboard';
server.use(NcToolGui.expressMiddleware(dashboardPath));
server.get('/', (_req, res) => res.redirect(dashboardPath));
return nestApp.getHttpAdapter().getInstance();
}
public static get httpServer(): http.Server {
return Noco._httpServer;
}
public static get server(): Express {
return Noco._server;
}
}

2
packages/nocodb-nest/src/app.module.ts

@ -56,7 +56,6 @@ import { GlobalModule } from './modules/global/global.module';
import { LocalStrategy } from './strategies/local.strategy';
import NcConfigFactory from './utils/NcConfigFactory';
import NcUpgrader from './version-upgrader/NcUpgrader';
import { ClientService } from './services/client/client.service';
import { AuthTokenStrategy } from './strategies/authtoken.strategy/authtoken.strategy';
import { BaseViewStrategy } from './strategies/base-view.strategy/base-view.strategy';
import { GoogleStrategy } from './strategies/google.strategy/google.strategy';
@ -146,7 +145,6 @@ export const JwtStrategyProvider: Provider = {
// JwtStrategyProvider,
LocalStrategy,
// ExtractProjectIdMiddleware,
ClientService,
AuthTokenStrategy,
BaseViewStrategy,
GoogleStrategy,

7
packages/nocodb-nest/src/modules/global/global.module.ts

@ -4,6 +4,7 @@ import { ExtractJwt } from 'passport-jwt'
import { Connection } from '../../connection/connection'
import { GlobalGuard } from '../../guards/global/global.guard'
import { MetaService } from '../../meta/meta.service'
import { SocketService } from '../../services/client/socket.service'
import { JwtStrategy } from '../../strategies/jwt.strategy'
import NcConfigFactory from '../../utils/NcConfigFactory'
import { jwtConstants } from '../auth/constants'
@ -39,7 +40,8 @@ export const JwtStrategyProvider: Provider = {
MetaService,
UsersService,
JwtStrategyProvider,
GlobalGuard
GlobalGuard,
SocketService,
],
exports: [
Connection,
@ -47,7 +49,8 @@ export const JwtStrategyProvider: Provider = {
// JwtService,
JwtStrategyProvider,
UsersService,
GlobalGuard
GlobalGuard,
SocketService,
],
})
export class GlobalModule {

24
packages/nocodb-nest/src/modules/import/import.controller.ts

@ -1,15 +1,16 @@
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common'
import { Controller, HttpCode, Post, Request, UseGuards } from '@nestjs/common';
import { forwardRef, Inject } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { GlobalGuard } from '../../guards/global/global.guard'
import { GlobalGuard } from '../../guards/global/global.guard';
import { NcError } from '../../helpers/catchError';
import { ExtractProjectIdMiddleware } from '../../middlewares/extract-project-id/extract-project-id.middleware'
import { ExtractProjectIdMiddleware } from '../../middlewares/extract-project-id/extract-project-id.middleware';
import { SyncSource } from '../../models';
import NocoJobs from '../../jobs/NocoJobs';
import { SocketService } from '../../services/client/socket.service';
import { ImportService } from './import.service';
import type { AirtableSyncConfig } from '../sync/helpers/job';
import airtableSyncJob from '../sync/helpers/job';
import type { Router } from 'express';
import type { Server } from 'socket.io';
const AIRTABLE_IMPORT_JOB = 'AIRTABLE_IMPORT_JOB';
@ -22,20 +23,16 @@ enum SyncStatus {
}
const jobs = [];
export default (
router: Router,
sv: Server,
jobs: { [id: string]: { last_message: any } },
) => {
const initJob = (sv: Server, jobs: { [p: string]: { last_message: any } }, socketService: SocketService) => {
// add importer job handler and progress notification job handler
NocoJobs.jobsMgr.addJobWorker(
AIRTABLE_IMPORT_JOB,
{} as any, // this?.syncService?.airtableImportJob,
airtableSyncJob
);
NocoJobs.jobsMgr.addJobWorker(
AIRTABLE_PROGRESS_JOB,
({ payload, progress }) => {
sv.to(payload?.id).emit('progress', {
socketService.io.to(payload?.id).emit('progress', {
msg: progress?.msg,
level: progress?.level,
status: progress?.status,
@ -87,6 +84,7 @@ export default (
export class ImportController {
constructor(
private readonly importService: ImportService,
private readonly socketService: SocketService,
@Inject(forwardRef(() => ModuleRef)) private readonly moduleRef: ModuleRef,
) {}
@ -149,4 +147,8 @@ export class ImportController {
}
return {};
}
async onModuleInit() {
initJob(this.socketService.io, this.socketService.jobs, this.socketService);
}
}

4
packages/nocodb-nest/src/modules/import/import.module.ts

@ -1,10 +1,12 @@
import { Module } from '@nestjs/common';
import { AttachmentsService } from '../attachments/attachments.service';
import { ColumnsService } from '../columns/columns.service';
import { BulkDataAliasService } from '../datas/bulk-data-alias/bulk-data-alias.service'
import { FiltersService } from '../filters/filters.service';
import { FormColumnsService } from '../form-columns/form-columns.service';
import { FormsService } from '../forms/forms.service';
import { GalleriesService } from '../galleries/galleries.service';
import { GlobalModule } from '../global/global.module';
import { GridsService } from '../grids/grids.service';
import { ProjectUsersService } from '../project-users/project-users.service';
import { ProjectsService } from '../projects/projects.service';
@ -16,6 +18,7 @@ import { ImportService } from './import.service';
import { ImportController } from './import.controller';
@Module({
imports: [GlobalModule],
controllers: [ImportController],
providers: [
ImportService,
@ -32,6 +35,7 @@ import { ImportController } from './import.controller';
ViewColumnsService,
SortsService,
GridsService,
BulkDataAliasService,
],
})
export class ImportModule {}

10
packages/nocodb-nest/src/modules/sync/helpers/job.ts

@ -10,6 +10,7 @@ import tinycolor from 'tinycolor2';
import extractRolesObj from '../../../utils/extractRolesObj';
import { AttachmentsService } from '../../attachments/attachments.service';
import { ColumnsService } from '../../columns/columns.service';
import { BulkDataAliasService } from '../../datas/bulk-data-alias/bulk-data-alias.service';
import { FiltersService } from '../../filters/filters.service';
import { FormColumnsService } from '../../form-columns/form-columns.service';
import { GalleriesService } from '../../galleries/galleries.service';
@ -112,6 +113,7 @@ export default async (
const viewColumnService = syncDB.moduleRef.get(ViewColumnsService);
const sortService = syncDB.moduleRef.get(SortsService);
const gridViewService = syncDB.moduleRef.get(GridsService);
const bulkDataService = syncDB.moduleRef.get(BulkDataAliasService);
const sMapEM = new EntityMap('aTblId', 'ncId', 'ncName', 'ncParent');
await sMapEM.init();
@ -2377,6 +2379,10 @@ export default async (
nocoBaseDataProcessing_v2,
sDB: syncDB,
logDetailed,
services: {
tableService,
bulkDataService,
},
});
rtc.data.records += await recordsMap[ncTbl.id].getCount();
@ -2410,6 +2416,10 @@ export default async (
atNcAliasRef,
ncLinkMappingTable,
syncDB,
services: {
bulkDataService,
tableService
}
});
}
} catch (error) {

36
packages/nocodb-nest/src/modules/sync/helpers/readAndProcessData.ts

@ -1,7 +1,8 @@
import { RelationTypes, UITypes } from 'nocodb-sdk';
// @ts-ignore
import { bulkDataService, tableService } from '../..';
import EntityMap from './EntityMap';
import type { BulkDataAliasService } from '../../datas/bulk-data-alias/bulk-data-alias.service';
import type { TablesService } from '../../tables/tables.service';
// @ts-ignore
import type { AirtableBase } from 'airtable/lib/airtable_base';
import type { TableType } from 'nocodb-sdk';
@ -9,17 +10,24 @@ const BULK_DATA_BATCH_SIZE = 500;
const ASSOC_BULK_DATA_BATCH_SIZE = 1000;
const BULK_PARALLEL_PROCESS = 5;
interface AirtableImportContext {
bulkDataService: BulkDataAliasService;
tableService: TablesService;
}
async function readAllData({
table,
fields,
base,
logBasic = (_str) => {},
services,
}: {
table: { title?: string };
fields?;
base: AirtableBase;
logBasic?: (string) => void;
logDetailed?: (string) => void;
services: AirtableImportContext;
}): Promise<EntityMap> {
return new Promise((resolve, reject) => {
let data = null;
@ -76,6 +84,7 @@ export async function importData({
sDB,
logDetailed = (_str) => {},
logBasic = (_str) => {},
services,
}: {
projectName: string;
table: { title?: string; id?: string };
@ -85,6 +94,7 @@ export async function importData({
logDetailed: (string) => void;
nocoBaseDataProcessing_v2;
sDB;
services: AirtableImportContext;
}): Promise<EntityMap> {
try {
// @ts-ignore
@ -117,7 +127,7 @@ export async function importData({
if (tempData.length >= BULK_DATA_BATCH_SIZE) {
let insertArray = tempData.splice(0, tempData.length);
await bulkDataService.bulkDataInsert({
await services.bulkDataService.bulkDataInsert({
projectName,
tableName: table.title,
body: insertArray,
@ -144,7 +154,7 @@ export async function importData({
readable.on('end', async () => {
await Promise.all(promises);
if (tempData.length > 0) {
await bulkDataService.bulkDataInsert({
await services.bulkDataService.bulkDataInsert({
projectName,
tableName: table.title,
body: tempData,
@ -185,6 +195,7 @@ export async function importLTARData({
atNcAliasRef,
ncLinkMappingTable,
syncDB,
services,
}: {
projectName: string;
table: { title?: string; id?: string };
@ -201,6 +212,7 @@ export async function importLTARData({
};
ncLinkMappingTable: Record<string, Record<string, any>>[];
syncDB;
services: AirtableImportContext;
}) {
const assocTableMetas: Array<{
modelMeta: { id?: string; title?: string };
@ -216,12 +228,14 @@ export async function importLTARData({
base,
logDetailed,
logBasic,
services
}));
const modelMeta: any = await tableService.getTableWithAccessibleViews({
tableId: table.id,
user: syncDB.user,
});
const modelMeta: any =
await services.tableService.getTableWithAccessibleViews({
tableId: table.id,
user: syncDB.user,
});
for (const colMeta of modelMeta.columns) {
// skip columns which are not LTAR and Many to many
@ -242,7 +256,7 @@ export async function importLTARData({
insertedAssocRef[colMeta.colOptions.fk_mm_model_id] = true;
const assocModelMeta: TableType =
(await tableService.getTableWithAccessibleViews({
(await services.tableService.getTableWithAccessibleViews({
tableId: colMeta.colOptions.fk_mm_model_id,
user: syncDB.user,
})) as any;
@ -299,7 +313,7 @@ export async function importLTARData({
)}`,
);
await bulkDataService.bulkDataInsert({
await services.bulkDataService.bulkDataInsert({
projectName,
tableName: assocMeta.modelMeta.title,
body: insertArray,
@ -327,7 +341,7 @@ export async function importLTARData({
)}`,
);
await bulkDataService.bulkDataInsert({
await services.bulkDataService.bulkDataInsert({
projectName,
tableName: assocMeta.modelMeta.title,
body: assocTableData,

13
packages/nocodb-nest/src/run/docker.ts

@ -1,7 +1,7 @@
import axios from 'axios';
import cors from 'cors';
import express from 'express';
import nocobuild from '../nocobuild';
import Noco from '../Noco'
const server = express();
server.enable('trust proxy');
@ -17,9 +17,16 @@ server.set('view engine', 'ejs');
process.env[`DEBUG`] = 'xc*';
// (async () => {
// await nocobuild(server);
// const httpServer = server.listen(process.env.PORT || 8080, async () => {
// console.log('Server started');
// });
// })().catch((e) => console.log(e));
(async () => {
await nocobuild(server);
const httpServer = server.listen(process.env.PORT || 8080, async () => {
console.log('Server started');
server.use(await Noco.init({}, httpServer, server));
});
})().catch((e) => console.log(e));

6
packages/nocodb-nest/src/run/dockerRunMysql.ts

@ -1,6 +1,6 @@
import cors from 'cors';
import express from 'express';
import nocobuild from '../nocobuild'
import Noco from '../Noco';
const server = express();
server.enable('trust proxy');
@ -27,8 +27,8 @@ process.env[`NC_DB`] = `mysql2://localhost:3306?u=root&p=password&d=${metaDb}`;
// process.env[`DEBUG`] = 'xc*';
(async () => {
await nocobuild(server);
const httpServer = server.listen(process.env.PORT || 8080, async () => {
console.log('Server started')
server.use(await Noco.init({}, httpServer, server));
});
})().catch((e) => console.log(e));

6
packages/nocodb-nest/src/run/dockerRunPG.ts

@ -1,6 +1,6 @@
import cors from 'cors';
import express from 'express';
import nocobuild from '../nocobuild'
import Noco from '../Noco';
const server = express();
server.enable('trust proxy');
@ -27,8 +27,8 @@ process.env[`NC_DB`] = `pg://localhost:5432?u=postgres&p=password&d=${metaDb}`;
// process.env[`DEBUG`] = 'xc*';
(async () => {
await nocobuild(server);
const httpServer = server.listen(process.env.PORT || 8080, async () => {
console.log('Server started')
server.use(await Noco.init({}, httpServer, server));
});
})().catch((e) => console.log(e));

5
packages/nocodb-nest/src/run/dockerRunPG_CyQuick.ts

@ -1,5 +1,6 @@
import cors from 'cors';
import express from 'express';
import Noco from '../Noco'
import nocobuild from '../nocobuild'
const server = express();
@ -20,8 +21,8 @@ process.env[
//process.env[`DEBUG`] = 'xc*';
(async () => {
await nocobuild(server);
const httpServer = server.listen(process.env.PORT || 8080, async () => {
console.log('Server started')
server.use(await Noco.init({}, httpServer, server));
});
})().catch((e) => console.log(e));

8
packages/nocodb-nest/src/run/testDocker.ts

@ -2,7 +2,7 @@ import axios from 'axios';
import cors from 'cors';
import express from 'express';
import { User } from '../models';
import nocobuild from '../nocobuild';
import Noco from '../Noco'
process.env.NC_VERSION = '0009044';
@ -21,17 +21,19 @@ server.set('view engine', 'ejs');
process.env[`DEBUG`] = 'xc*';
(async () => {
await nocobuild(server);
const httpServer = server.listen(process.env.PORT || 8080, async () => {
server.use(await Noco.init({}, httpServer, server));
if (!(await User.getByEmail('user@nocodb.com'))) {
const response = await axios.post(
`http://localhost:${process.env.PORT || 8080}/api/v1/auth/user/signup`,
{
email: 'user@nocodb.com',
password: 'Password123.',
},
}
);
console.log(response.data);
}
});
})().catch((e) => console.log(e));

83
packages/nocodb-nest/src/services/client/client.service.ts

@ -1,83 +0,0 @@
import crypto from 'crypto';
import { Inject, Injectable, UnauthorizedException } from '@nestjs/common';
import { HttpAdapterHost } from '@nestjs/core';
import { T } from 'nc-help';
import { Server } from 'socket.io';
import { AuthGuard } from '@nestjs/passport';
import { JwtStrategy } from '../../strategies/jwt.strategy';
import type { OnModuleInit } from '@nestjs/common';
import type { Socket } from 'socket.io';
import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
function getHash(str) {
return crypto.createHash('md5').update(str).digest('hex');
}
@Injectable()
export class ClientService implements OnModuleInit {
// private server: HttpServer;
private clients: { [id: string]: Socket } = {};
private jobs: { [id: string]: { last_message: any } } = {};
constructor(
private jwtStrategy: JwtStrategy,
@Inject(HttpAdapterHost) private httpAdapterHost: HttpAdapterHost,
) {
}
async onModuleInit() {
const io = new Server(this.httpAdapterHost.httpAdapter.getHttpServer(), {
cors: {
origin: '*',
allowedHeaders: ['xc-auth'],
credentials: true,
},
});
io.use(async (socket, next) => {
// const authGuard = new (AuthGuard('jwt'))();
// const result = await authGuard.canActivate(socket.handshake as any);
// if (!result) {
// throw new UnauthorizedException();
// }
// return new Promise((resolve, reject) => {
// this.jwtStrategy.authenticate(
// socket.handshake as any,
// (error, user) => {
// if (error) {
// reject(new UnauthorizedException(error.message));
// } else {
// resolve(user);
// }
// },
// );
// });
try {
const context = new ExecutionContextHost([socket.handshake as any]);
const guard = new (AuthGuard('jwt'))(context);
const canActivate = await guard.canActivate(context);
} catch {}
next()
}).on('connection', (socket) => {
this.clients[socket.id] = socket;
const id = getHash(
(process.env.NC_SERVER_UUID || T.id) +
(socket?.handshake as any)?.user?.id,
);
socket.on('page', (args) => {
T.page({ ...args, id });
});
socket.on('event', (args) => {
T.event({ ...args, id });
});
socket.on('subscribe', (room) => {
if (room in this.jobs) {
socket.join(room);
socket.emit('job');
socket.emit('progress', this.jobs[room].last_message);
}
});
});
}
}

8
packages/nocodb-nest/src/services/client/client.service.spec.ts → packages/nocodb-nest/src/services/client/socket.service.spec.ts

@ -1,15 +1,15 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ClientService } from './client.service';
import { SocketService } from './socket.service';
describe('ClientService', () => {
let service: ClientService;
let service: SocketService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ClientService],
providers: [SocketService],
}).compile();
service = module.get<ClientService>(ClientService);
service = module.get<SocketService>(SocketService);
});
it('should be defined', () => {

81
packages/nocodb-nest/src/services/client/socket.service.ts

@ -0,0 +1,81 @@
import crypto from 'crypto';
import { Inject, Injectable, UnauthorizedException } from '@nestjs/common';
import { HttpAdapterHost } from '@nestjs/core';
import { T } from 'nc-help';
import { Server } from 'socket.io';
import { AuthGuard } from '@nestjs/passport';
import { ExecutionContextHost } from '@nestjs/core/helpers/execution-context-host';
import Noco from '../../Noco';
import { JwtStrategy } from '../../strategies/jwt.strategy';
import type { OnModuleInit } from '@nestjs/common';
import type { Socket } from 'socket.io';
function getHash(str) {
return crypto.createHash('md5').update(str).digest('hex');
}
@Injectable()
export class SocketService implements OnModuleInit {
// private server: HttpServer;
private clients: { [id: string]: Socket } = {};
private _jobs: { [id: string]: { last_message: any } } = {};
private _io: Server;
constructor(
private jwtStrategy: JwtStrategy,
@Inject(HttpAdapterHost) private httpAdapterHost: HttpAdapterHost,
) {}
async onModuleInit() {
console.log(Noco.httpServer)
this._io = new Server(
Noco.httpServer ?? this.httpAdapterHost.httpAdapter.getHttpServer(),
{
cors: {
origin: '*',
allowedHeaders: ['xc-auth'],
credentials: true,
},
},
);
this.io
.use(async (socket, next) => {
try {
const context = new ExecutionContextHost([socket.handshake as any]);
const guard = new (AuthGuard('jwt'))(context);
await guard.canActivate(context);
} catch {}
next();
})
.on('connection', (socket) => {
this.clients[socket.id] = socket;
const id = getHash(
(process.env.NC_SERVER_UUID || T.id) +
(socket?.handshake as any)?.user?.id,
);
socket.on('page', (args) => {
T.page({ ...args, id });
});
socket.on('event', (args) => {
T.event({ ...args, id });
});
socket.on('subscribe', (room) => {
if (room in this.jobs) {
socket.join(room);
socket.emit('job');
socket.emit('progress', this.jobs[room].last_message);
}
});
});
}
public get io() {
return this._io;
}
public get jobs() {
return this._jobs;
}
}
Loading…
Cancel
Save