mirror of https://github.com/nocodb/nocodb
Browse Source
* feat: clean-up job * feat: source cleanup * fix: remove unused method * feat: move jobs-redis to a static class * feat: release sources from in-memory db on update & delete * fix: skip calls if job redis not available * fix: error handling on connection delete --------- Co-authored-by: mertmit <mertmit99@gmail.com>pull/8557/head
Raju Udava
7 months ago
committed by
GitHub
9 changed files with 238 additions and 160 deletions
@ -1,91 +0,0 @@
|
||||
import { Injectable, Logger } from '@nestjs/common'; |
||||
import Redis from 'ioredis'; |
||||
import { InstanceTypes } from '~/interface/Jobs'; |
||||
|
||||
@Injectable() |
||||
export class JobsRedisService { |
||||
protected logger = new Logger(JobsRedisService.name); |
||||
|
||||
private redisClient: Redis; |
||||
private redisSubscriber: Redis; |
||||
private unsubscribeCallbacks: { [key: string]: () => void } = {}; |
||||
|
||||
public primaryCallbacks: { [key: string]: (...args) => void } = {}; |
||||
public workerCallbacks: { [key: string]: (...args) => void } = {}; |
||||
|
||||
constructor() { |
||||
this.redisClient = new Redis(process.env.NC_REDIS_JOB_URL); |
||||
this.redisSubscriber = new Redis(process.env.NC_REDIS_JOB_URL); |
||||
|
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
this.redisSubscriber.subscribe(InstanceTypes.WORKER); |
||||
} else { |
||||
this.redisSubscriber.subscribe(InstanceTypes.PRIMARY); |
||||
} |
||||
|
||||
const onMessage = (channel, message) => { |
||||
const args = message.split(':'); |
||||
const command = args.shift(); |
||||
if (channel === InstanceTypes.WORKER) { |
||||
this.workerCallbacks[command] && this.workerCallbacks[command](...args); |
||||
} else if (channel === InstanceTypes.PRIMARY) { |
||||
this.primaryCallbacks[command] && |
||||
this.primaryCallbacks[command](...args); |
||||
} |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
} |
||||
|
||||
publish(channel: string, message: string | any) { |
||||
if (typeof message === 'string') { |
||||
this.redisClient.publish(channel, message); |
||||
} else { |
||||
try { |
||||
this.redisClient.publish(channel, JSON.stringify(message)); |
||||
} catch (e) { |
||||
this.logger.error(e); |
||||
} |
||||
} |
||||
} |
||||
|
||||
subscribe(channel: string, callback: (message: any) => void) { |
||||
this.redisSubscriber.subscribe(channel); |
||||
|
||||
const onMessage = (_channel, message) => { |
||||
try { |
||||
message = JSON.parse(message); |
||||
} catch (e) {} |
||||
callback(message); |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
this.unsubscribeCallbacks[channel] = () => { |
||||
this.redisSubscriber.unsubscribe(channel); |
||||
this.redisSubscriber.off('message', onMessage); |
||||
}; |
||||
} |
||||
|
||||
unsubscribe(channel: string) { |
||||
if (this.unsubscribeCallbacks[channel]) { |
||||
this.unsubscribeCallbacks[channel](); |
||||
delete this.unsubscribeCallbacks[channel]; |
||||
} |
||||
} |
||||
|
||||
workerCount(): Promise<number> { |
||||
return new Promise((resolve, reject) => { |
||||
this.redisClient.publish( |
||||
InstanceTypes.WORKER, |
||||
'count', |
||||
(error, numberOfSubscribers) => { |
||||
if (error) { |
||||
reject(0); |
||||
} else { |
||||
resolve(numberOfSubscribers); |
||||
} |
||||
}, |
||||
); |
||||
}); |
||||
} |
||||
} |
||||
@ -0,0 +1,155 @@
|
||||
import { Logger } from '@nestjs/common'; |
||||
import Redis from 'ioredis'; |
||||
import type { InstanceCommands } from '~/interface/Jobs'; |
||||
import { InstanceTypes } from '~/interface/Jobs'; |
||||
|
||||
export class JobsRedis { |
||||
private static initialized = false; |
||||
|
||||
public static available = process.env.NC_REDIS_JOB_URL ? true : false; |
||||
|
||||
protected static logger = new Logger(JobsRedis.name); |
||||
|
||||
private static redisClient: Redis; |
||||
private static redisSubscriber: Redis; |
||||
private static unsubscribeCallbacks: { [key: string]: () => Promise<void> } = |
||||
{}; |
||||
|
||||
public static primaryCallbacks: { |
||||
[key: string]: (...args) => Promise<void>; |
||||
} = {}; |
||||
public static workerCallbacks: { [key: string]: (...args) => Promise<void> } = |
||||
{}; |
||||
|
||||
static async init() { |
||||
if (this.initialized) { |
||||
return; |
||||
} |
||||
|
||||
if (!JobsRedis.available) { |
||||
return; |
||||
} |
||||
|
||||
this.initialized = true; |
||||
|
||||
this.redisClient = new Redis(process.env.NC_REDIS_JOB_URL); |
||||
this.redisSubscriber = new Redis(process.env.NC_REDIS_JOB_URL); |
||||
|
||||
if (process.env.NC_WORKER_CONTAINER === 'true') { |
||||
await this.redisSubscriber.subscribe(InstanceTypes.WORKER); |
||||
} else { |
||||
await this.redisSubscriber.subscribe(InstanceTypes.PRIMARY); |
||||
} |
||||
|
||||
const onMessage = async (channel, message) => { |
||||
const args = message.split(':'); |
||||
const command = args.shift(); |
||||
if (channel === InstanceTypes.WORKER) { |
||||
this.workerCallbacks[command] && |
||||
(await this.workerCallbacks[command](...args)); |
||||
} else if (channel === InstanceTypes.PRIMARY) { |
||||
this.primaryCallbacks[command] && |
||||
(await this.primaryCallbacks[command](...args)); |
||||
} |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
} |
||||
|
||||
static async publish(channel: string, message: string | any) { |
||||
if (!this.initialized) { |
||||
if (!JobsRedis.available) { |
||||
return; |
||||
} |
||||
|
||||
await this.init(); |
||||
} |
||||
|
||||
if (typeof message === 'string') { |
||||
await this.redisClient.publish(channel, message); |
||||
} else { |
||||
try { |
||||
await this.redisClient.publish(channel, JSON.stringify(message)); |
||||
} catch (e) { |
||||
this.logger.error(e); |
||||
} |
||||
} |
||||
} |
||||
|
||||
static async subscribe( |
||||
channel: string, |
||||
callback: (message: any) => Promise<void>, |
||||
) { |
||||
if (!this.initialized) { |
||||
if (!JobsRedis.available) { |
||||
return; |
||||
} |
||||
|
||||
await this.init(); |
||||
} |
||||
|
||||
await this.redisSubscriber.subscribe(channel); |
||||
|
||||
const onMessage = async (_channel, message) => { |
||||
try { |
||||
message = JSON.parse(message); |
||||
} catch (e) {} |
||||
await callback(message); |
||||
}; |
||||
|
||||
this.redisSubscriber.on('message', onMessage); |
||||
this.unsubscribeCallbacks[channel] = async () => { |
||||
await this.redisSubscriber.unsubscribe(channel); |
||||
this.redisSubscriber.off('message', onMessage); |
||||
}; |
||||
} |
||||
|
||||
static async unsubscribe(channel: string) { |
||||
if (!this.initialized) { |
||||
if (!JobsRedis.available) { |
||||
return; |
||||
} |
||||
|
||||
await this.init(); |
||||
} |
||||
|
||||
if (this.unsubscribeCallbacks[channel]) { |
||||
await this.unsubscribeCallbacks[channel](); |
||||
delete this.unsubscribeCallbacks[channel]; |
||||
} |
||||
} |
||||
|
||||
static async workerCount(): Promise<number> { |
||||
if (!this.initialized) { |
||||
if (!JobsRedis.available) { |
||||
return; |
||||
} |
||||
|
||||
await this.init(); |
||||
} |
||||
|
||||
return new Promise((resolve, reject) => { |
||||
this.redisClient.publish( |
||||
InstanceTypes.WORKER, |
||||
'count', |
||||
(error, numberOfSubscribers) => { |
||||
if (error) { |
||||
reject(0); |
||||
} else { |
||||
resolve(numberOfSubscribers); |
||||
} |
||||
}, |
||||
); |
||||
}); |
||||
} |
||||
|
||||
static async emitWorkerCommand(command: InstanceCommands, ...args: any[]) { |
||||
const data = `${command}${args.length ? `:${args.join(':')}` : ''}`; |
||||
await JobsRedis.publish(InstanceTypes.WORKER, data); |
||||
} |
||||
|
||||
static async emitPrimaryCommand(command: InstanceCommands, ...args: any[]) { |
||||
const data = `${command}${args.length ? `:${args.join(':')}` : ''}`; |
||||
await JobsRedis.publish(InstanceTypes.PRIMARY, data); |
||||
} |
||||
} |
||||
Loading…
Reference in new issue