From 9e0fe68ea51d8d62dc703b766a086a1de16569f2 Mon Sep 17 00:00:00 2001 From: mertmit Date: Fri, 19 Jan 2024 14:19:26 +0000 Subject: [PATCH] fix: keep list of parents on child object cache --- packages/nocodb/src/cache/RedisCacheMgr.ts | 92 +++++++++++++++++- .../nocodb/src/cache/RedisMockCacheMgr.ts | 94 ++++++++++++++++++- packages/nocodb/src/utils/globals.ts | 2 + 3 files changed, 182 insertions(+), 6 deletions(-) diff --git a/packages/nocodb/src/cache/RedisCacheMgr.ts b/packages/nocodb/src/cache/RedisCacheMgr.ts index 5bf07a5b81..542edfc6ff 100644 --- a/packages/nocodb/src/cache/RedisCacheMgr.ts +++ b/packages/nocodb/src/cache/RedisCacheMgr.ts @@ -1,7 +1,11 @@ import debug from 'debug'; import Redis from 'ioredis'; import CacheMgr from './CacheMgr'; -import { CacheDelDirection, CacheGetType } from '~/utils/globals'; +import { + CacheDelDirection, + CacheGetType, + CacheListProp, +} from '~/utils/globals'; const log = debug('nc:cache'); @@ -91,6 +95,10 @@ export default class RedisCacheMgr extends CacheMgr { if (Array.isArray(value) && value.length) { return this.client.sadd(key, value); } + const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT); + if (keyValue) { + value = await this.prepareValue(value, this.getParents(keyValue)); + } return this.client.set( key, JSON.stringify(value, this.getCircularReplacer()), @@ -195,9 +203,19 @@ export default class RedisCacheMgr extends CacheMgr { // e.g. nc:::: getKey = `${this.prefix}:${scope}:${propValues.join(':')}`; } + log(`RedisCacheMgr::setList: get key ${getKey}`); + // get Get Key + let value = await this.get(getKey, CacheGetType.TYPE_OBJECT); + if (value) { + log(`RedisCacheMgr::setList: preparing key ${getKey}`); + // prepare Get Key + value = await this.prepareValue(o, this.getParents(value), listKey); + } else { + value = await this.prepareValue(o, [], listKey); + } // set Get Key log(`RedisCacheMgr::setList: setting key ${getKey}`); - await this.set(getKey, JSON.stringify(o, this.getCircularReplacer())); + await this.set(getKey, JSON.stringify(value, this.getCircularReplacer())); // push Get Key to List listOfGetKeys.push(getKey); } @@ -213,8 +231,9 @@ export default class RedisCacheMgr extends CacheMgr { ): Promise { log(`RedisCacheMgr::deepDel: choose direction ${direction}`); if (direction === CacheDelDirection.CHILD_TO_PARENT) { + const childKey = await this.get(key, CacheGetType.TYPE_OBJECT); // given a child key, delete all keys in corresponding parent lists - const scopeList = await this.client.keys(`${this.prefix}:${scope}*list`); + const scopeList = this.getParents(childKey); for (const listKey of scopeList) { // get target list let list = (await this.get(listKey, CacheGetType.TYPE_ARRAY)) || []; @@ -271,10 +290,77 @@ export default class RedisCacheMgr extends CacheMgr { list = []; await this.del(listKey); } + + log(`RedisCacheMgr::appendToList: get key ${key}`); + // get Get Key + const value = await this.get(key, CacheGetType.TYPE_OBJECT); + log(`RedisCacheMgr::appendToList: preparing key ${key}`); + if (!value) { + console.error(`RedisCacheMgr::appendToList: value is empty for ${key}`); + await this.del(listKey); + return false; + } + // prepare Get Key + const preparedValue = await this.prepareValue( + value, + this.getParents(value), + listKey, + ); + // set Get Key + log(`RedisCacheMgr::appendToList: setting key ${key}`); + await this.set( + key, + JSON.stringify(preparedValue, this.getCircularReplacer()), + ); + list.push(key); return this.set(listKey, list); } + prepareValue(value, listKeys = [], newParent?) { + if (newParent) { + listKeys.push(newParent); + } + + if (value && typeof value === 'object') { + value[CacheListProp] = listKeys; + } else if (value && typeof value === 'string') { + const keyHelper = value.split(CacheListProp); + if (listKeys.length) { + value = `${keyHelper[0]}${CacheListProp}${listKeys.join(',')}`; + } + } else if (value) { + console.error( + `RedisCacheMgr::prepareListKey: keyValue is not object or string`, + value, + ); + throw new Error( + `RedisCacheMgr::prepareListKey: keyValue is not object or string`, + ); + } + return value; + } + + getParents(value) { + if (value && typeof value === 'object') { + if (CacheListProp in value) { + const listsForKey = value[CacheListProp]; + if (listsForKey && listsForKey.length) { + return listsForKey; + } + } + } else if (value && typeof value === 'string') { + if (value.includes(CacheListProp)) { + const keyHelper = value.split(CacheListProp); + const listsForKey = keyHelper[1].split(','); + if (listsForKey.length) { + return listsForKey; + } + } + } + return []; + } + async destroy(): Promise { log('RedisCacheMgr::destroy: destroy redis'); return this.client.flushdb().then((r) => r === 'OK'); diff --git a/packages/nocodb/src/cache/RedisMockCacheMgr.ts b/packages/nocodb/src/cache/RedisMockCacheMgr.ts index e134ce2e85..8c3ac4f095 100644 --- a/packages/nocodb/src/cache/RedisMockCacheMgr.ts +++ b/packages/nocodb/src/cache/RedisMockCacheMgr.ts @@ -2,7 +2,11 @@ import debug from 'debug'; import Redis from 'ioredis-mock'; import CacheMgr from './CacheMgr'; import type IORedis from 'ioredis'; -import { CacheDelDirection, CacheGetType } from '~/utils/globals'; +import { + CacheDelDirection, + CacheGetType, + CacheListProp, +} from '~/utils/globals'; const log = debug('nc:cache'); export default class RedisMockCacheMgr extends CacheMgr { @@ -84,6 +88,10 @@ export default class RedisMockCacheMgr extends CacheMgr { if (Array.isArray(value) && value.length) { return this.client.sadd(key, value); } + const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT); + if (keyValue) { + value = await this.prepareValue(value, this.getParents(keyValue)); + } return this.client.set( key, JSON.stringify(value, this.getCircularReplacer()), @@ -192,9 +200,19 @@ export default class RedisMockCacheMgr extends CacheMgr { // e.g. nc:::: getKey = `${this.prefix}:${scope}:${propValues.join(':')}`; } + log(`RedisMockCacheMgr::setList: get key ${getKey}`); + // get Get Key + let value = await this.get(getKey, CacheGetType.TYPE_OBJECT); + if (value) { + log(`RedisMockCacheMgr::setList: preparing key ${getKey}`); + // prepare Get Key + value = await this.prepareValue(o, this.getParents(value), listKey); + } else { + value = await this.prepareValue(o, [], listKey); + } // set Get Key log(`RedisMockCacheMgr::setList: setting key ${getKey}`); - await this.set(getKey, JSON.stringify(o, this.getCircularReplacer())); + await this.set(getKey, JSON.stringify(value, this.getCircularReplacer())); // push Get Key to List listOfGetKeys.push(getKey); } @@ -210,8 +228,9 @@ export default class RedisMockCacheMgr extends CacheMgr { ): Promise { log(`RedisMockCacheMgr::deepDel: choose direction ${direction}`); if (direction === CacheDelDirection.CHILD_TO_PARENT) { + const childKey = await this.get(key, CacheGetType.TYPE_OBJECT); // given a child key, delete all keys in corresponding parent lists - const scopeList = await this.client.keys(`${this.prefix}:${scope}*list`); + const scopeList = this.getParents(childKey); for (const listKey of scopeList) { // get target list let list = (await this.get(listKey, CacheGetType.TYPE_ARRAY)) || []; @@ -268,10 +287,79 @@ export default class RedisMockCacheMgr extends CacheMgr { list = []; await this.del(listKey); } + + log(`RedisMockCacheMgr::appendToList: get key ${key}`); + // get Get Key + const value = await this.get(key, CacheGetType.TYPE_OBJECT); + log(`RedisMockCacheMgr::appendToList: preparing key ${key}`); + if (!value) { + console.error( + `RedisMockCacheMgr::appendToList: value is empty for ${key}`, + ); + await this.del(listKey); + return false; + } + // prepare Get Key + const preparedValue = await this.prepareValue( + value, + this.getParents(value), + listKey, + ); + // set Get Key + log(`RedisMockCacheMgr::appendToList: setting key ${key}`); + await this.set( + key, + JSON.stringify(preparedValue, this.getCircularReplacer()), + ); + list.push(key); return this.set(listKey, list); } + prepareValue(value, listKeys = [], newParent?) { + if (newParent) { + listKeys.push(newParent); + } + + if (value && typeof value === 'object') { + value[CacheListProp] = listKeys; + } else if (value && typeof value === 'string') { + const keyHelper = value.split(CacheListProp); + if (listKeys.length) { + value = `${keyHelper[0]}${CacheListProp}${listKeys.join(',')}`; + } + } else if (value) { + console.error( + `RedisMockCacheMgr::prepareListKey: keyValue is not object or string`, + value, + ); + throw new Error( + `RedisMockCacheMgr::prepareListKey: keyValue is not object or string`, + ); + } + return value; + } + + getParents(value) { + if (value && typeof value === 'object') { + if (CacheListProp in value) { + const listsForKey = value[CacheListProp]; + if (listsForKey && listsForKey.length) { + return listsForKey; + } + } + } else if (value && typeof value === 'string') { + if (value.includes(CacheListProp)) { + const keyHelper = value.split(CacheListProp); + const listsForKey = keyHelper[1].split(','); + if (listsForKey.length) { + return listsForKey; + } + } + } + return []; + } + async destroy(): Promise { log('RedisMockCacheMgr::destroy: destroy redis'); return this.client.flushdb().then((r) => r === 'OK'); diff --git a/packages/nocodb/src/utils/globals.ts b/packages/nocodb/src/utils/globals.ts index 9d9f712241..37d310ea18 100644 --- a/packages/nocodb/src/utils/globals.ts +++ b/packages/nocodb/src/utils/globals.ts @@ -175,6 +175,8 @@ export enum CacheDelDirection { CHILD_TO_PARENT = 'CHILD_TO_PARENT', } +export const CacheListProp = '__nc_list__'; + export const GROUPBY_COMPARISON_OPS = [ // these are used for groupby 'gb_eq',