github
/
nocodb
mirror of https://github.com/nocodb/nocodb
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

feat: wrap all cache with metadata

pull/7596/head
mertmit 10 months ago
parent
d87b8f65cb
commit
558beb4fb2
2 changed files with 130 additions and 185 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 245
      packages/nocodb/src/cache/CacheMgr.ts
  2. 6
      packages/nocodb/src/utils/globals.ts

245
packages/nocodb/src/cache/CacheMgr.ts vendored
Unescape View File

@ -1,14 +1,10 @@
import debug from 'debug'; import debug from 'debug';
import { Logger } from '@nestjs/common';
import type IORedis from 'ioredis'; import type IORedis from 'ioredis';
import { import { CacheDelDirection, CacheGetType } from '~/utils/globals';
CacheDelDirection,
CacheGetType,
CacheListProp,
CacheMetaSplitter,
CacheTimestampProp,
} from '~/utils/globals';
const log = debug('nc:cache'); const log = debug('nc:cache');
const logger = new Logger('CacheMgr');
export default abstract class CacheMgr { export default abstract class CacheMgr {
client: IORedis; client: IORedis;
prefix: string; prefix: string;
@ -41,12 +37,13 @@ export default abstract class CacheMgr {
} }
// @ts-ignore // @ts-ignore
async get(key: string, type: string): Promise<any> { private async getRaw(key: string, type?: string): Promise<any> {
log(`${this.context}::get: getting key ${key} with type ${type}`); log(`${this.context}::getRaw: getting key ${key} with type ${type}`);
if (type === CacheGetType.TYPE_ARRAY) { if (type === CacheGetType.TYPE_ARRAY) {
return this.client.smembers(key); return this.client.smembers(key);
} else if (type === CacheGetType.TYPE_OBJECT) { } else {
const res = await this.client.get(key); const res = await this.client.get(key);
if (res) {
try { try {
const o = JSON.parse(res); const o = JSON.parse(res);
if (typeof o === 'object') { if (typeof o === 'object') {
@ -59,20 +56,23 @@ export default abstract class CacheMgr {
} }
return Promise.resolve(o); return Promise.resolve(o);
} }
} catch (e) {} } catch (e) {
const valueHelper = res.split(CacheMetaSplitter); logger.error(`Bad value stored for key ${key} : ${res}`);
return Promise.resolve(valueHelper[0]); return Promise.resolve(res);
} else if (type === CacheGetType.TYPE_STRING) {
return this.client.get(key).then((res) => {
if (!res) {
return res;
} }
const valueHelper = res.split(CacheMetaSplitter);
return valueHelper[0];
});
} }
log(`Invalid CacheGetType: ${type}`); return Promise.resolve(res);
return Promise.resolve(false); }
}
// @ts-ignore
async get(key: string, type: string): Promise<any> {
return this.getRaw(key, type).then((res) => {
if (res && res.value) {
return res.value;
}
return res;
});
} }
// @ts-ignore // @ts-ignore
@ -80,7 +80,9 @@ export default abstract class CacheMgr {
key: string, key: string,
value: any, value: any,
options: { options: {
// when we prepare beforehand, we don't need to prepare again
skipPrepare?: boolean; skipPrepare?: boolean;
// timestamp for the value, if not provided, it will be set to current time
timestamp?: number; timestamp?: number;
} = { } = {
skipPrepare: false, skipPrepare: false,
@ -91,16 +93,16 @@ export default abstract class CacheMgr {
if (typeof value !== 'undefined' && value) { if (typeof value !== 'undefined' && value) {
log(`${this.context}::set: setting key ${key} with value ${value}`); log(`${this.context}::set: setting key ${key} with value ${value}`);
if (typeof value === 'object') {
if (Array.isArray(value) && value.length) { if (Array.isArray(value) && value.length) {
return this.client.sadd(key, value); return this.client.sadd(key, value);
} }
if (!skipPrepare) { if (!skipPrepare) {
// try to get old key value // try to get old key value
const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT); const keyValue = await this.getRaw(key);
// prepare new key value // prepare new key value
value = this.prepareValue(value, { value = this.prepareValue({
value,
parentKeys: this.getParents(keyValue), parentKeys: this.getParents(keyValue),
timestamp, timestamp,
}); });
@ -110,19 +112,6 @@ export default abstract class CacheMgr {
key, key,
JSON.stringify(value, this.getCircularReplacer()), JSON.stringify(value, this.getCircularReplacer()),
); );
}
if (!skipPrepare) {
// try to get old key value
const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT);
// prepare new key value
value = this.prepareValue(value.toString(), {
parentKeys: this.getParents(keyValue),
timestamp,
});
}
return this.client.set(key, value);
} else { } else {
log(`${this.context}::set: value is empty for ${key}. Skipping ...`); log(`${this.context}::set: value is empty for ${key}. Skipping ...`);
return Promise.resolve(true); return Promise.resolve(true);
@ -130,23 +119,47 @@ export default abstract class CacheMgr {
} }
// @ts-ignore // @ts-ignore
async setExpiring(key: string, value: any, seconds: number): Promise<any> { async setExpiring(
key: string,
value: any,
seconds: number,
options: {
// when we prepare beforehand, we don't need to prepare again
skipPrepare?: boolean;
// timestamp for the value, if not provided, it will be set to current time
timestamp?: number;
} = {
skipPrepare: false,
},
): Promise<any> {
const { skipPrepare, timestamp } = options;
if (typeof value !== 'undefined' && value) { if (typeof value !== 'undefined' && value) {
log( log(
`${this.context}::setExpiring: setting key ${key} with value ${value} for ${seconds} seconds`, `${this.context}::setExpiring: setting key ${key} with value ${value}`,
); );
if (typeof value === 'object') {
if (Array.isArray(value) && value.length) { if (Array.isArray(value) && value.length) {
return this.client.sadd(key, value); return this.client.sadd(key, value);
} }
if (!skipPrepare) {
// try to get old key value
const keyValue = await this.getRaw(key);
// prepare new key value
value = this.prepareValue({
value,
parentKeys: this.getParents(keyValue),
timestamp,
});
}
return this.client.set( return this.client.set(
key, key,
JSON.stringify(value, this.getCircularReplacer()), JSON.stringify(value, this.getCircularReplacer()),
'EX', 'EX',
seconds, seconds,
); );
}
return this.client.set(key, value, 'EX', seconds);
} else { } else {
log(`${this.context}::set: value is empty for ${key}. Skipping ...`); log(`${this.context}::set: value is empty for ${key}. Skipping ...`);
return Promise.resolve(true); return Promise.resolve(true);
@ -189,7 +202,7 @@ export default abstract class CacheMgr {
if (values.some((v) => v === null)) { if (values.some((v) => v === null)) {
// FALLBACK: a key is missing from list, this should never happen // FALLBACK: a key is missing from list, this should never happen
console.error(`${this.context}::getList: missing value for ${key}`); logger.error(`${this.context}::getList: missing value for ${key}`);
const allParents = []; const allParents = [];
// get all parents from children // get all parents from children
values.forEach((v) => { values.forEach((v) => {
@ -214,7 +227,7 @@ export default abstract class CacheMgr {
try { try {
const o = JSON.parse(res); const o = JSON.parse(res);
if (typeof o === 'object') { if (typeof o === 'object') {
return o; return o.value;
} }
} catch (e) { } catch (e) {
return res; return res;
@ -260,30 +273,29 @@ export default abstract class CacheMgr {
} }
log(`${this.context}::setList: get key ${getKey}`); log(`${this.context}::setList: get key ${getKey}`);
// get key // get key
let value = await this.get(getKey, CacheGetType.TYPE_OBJECT); let value = await this.getRaw(getKey, CacheGetType.TYPE_OBJECT);
if (value) { if (value) {
log(`${this.context}::setList: preparing key ${getKey}`); log(`${this.context}::setList: preparing key ${getKey}`);
// prepare key // prepare key
value = this.prepareValue(o, { value = this.prepareValue({
parentKeys: this.getParents(value).concat(listKey), value: o,
parentKeys: this.getParents(value),
newKey: listKey,
timestamp, timestamp,
}); });
} else { } else {
value = this.prepareValue(o, { value = this.prepareValue({
value: o,
parentKeys: [listKey], parentKeys: [listKey],
timestamp, timestamp,
}); });
} }
// set key // set key
log(`${this.context}::setList: setting key ${getKey}`); log(`${this.context}::setList: setting key ${getKey}`);
await this.set( await this.set(getKey, value, {
getKey,
JSON.stringify(value, this.getCircularReplacer()),
{
skipPrepare: true, skipPrepare: true,
timestamp, timestamp,
}, });
);
// push key to list // push key to list
listOfGetKeys.push(getKey); listOfGetKeys.push(getKey);
} }
@ -361,7 +373,7 @@ export default abstract class CacheMgr {
log(`${this.context}::appendToList: preparing key ${key}`); log(`${this.context}::appendToList: preparing key ${key}`);
if (!value) { if (!value) {
// FALLBACK: this is to get rid of all keys that would be effected by this (should never happen) // FALLBACK: this is to get rid of all keys that would be effected by this (should never happen)
console.error(`${this.context}::appendToList: value is empty for ${key}`); logger.error(`${this.context}::appendToList: value is empty for ${key}`);
const allParents = []; const allParents = [];
// get all children // get all children
const listValues = await this.getList(scope, subListKeys); const listValues = await this.getList(scope, subListKeys);
@ -380,115 +392,54 @@ export default abstract class CacheMgr {
return false; return false;
} }
// prepare Get Key // prepare Get Key
const preparedValue = this.prepareValue(value, { const preparedValue = this.prepareValue({
parentKeys: this.getParents(value).concat(listKey), value,
parentKeys: this.getParents(value),
newKey: listKey,
}); });
// set Get Key // set Get Key
log(`${this.context}::appendToList: setting key ${key}`); log(`${this.context}::appendToList: setting key ${key}`);
await this.set( await this.set(key, preparedValue, {
key,
JSON.stringify(preparedValue, this.getCircularReplacer()),
{
skipPrepare: true, skipPrepare: true,
}, });
);
list.push(key); list.push(key);
return this.set(listKey, list); return this.set(listKey, list);
} }
prepareValue( // wrap value with metadata
value, prepareValue(args: {
options: { value: any;
parentKeys: string[]; parentKeys: string[];
newKey?: string;
timestamp?: number; timestamp?: number;
}, }) {
) { const { value, parentKeys, newKey, timestamp } = args;
const { parentKeys, timestamp } = options;
if (value && typeof value === 'object') {
value[CacheListProp] = parentKeys;
if (timestamp) { if (newKey && !parentKeys.includes(newKey)) {
value[CacheTimestampProp] = timestamp; parentKeys.push(newKey);
} else {
value[CacheTimestampProp] = Date.now();
}
} else if (value && typeof value === 'string') {
const metaHelper = value.split(CacheMetaSplitter);
if (metaHelper.length > 1) {
const keyVal = metaHelper[0];
const keyMeta = metaHelper[1];
try {
const meta = JSON.parse(keyMeta);
meta[CacheListProp] = parentKeys;
meta[CacheTimestampProp] = timestamp || Date.now();
value = `${keyVal}${CacheMetaSplitter}${JSON.stringify(meta)}`;
} catch (e) {
console.error(
`${this.context}::prepareValue: keyValue meta is not JSON`,
keyMeta,
);
throw new Error(
`${this.context}::prepareValue: keyValue meta is not JSON`,
);
}
} else {
const meta = {
[CacheListProp]: parentKeys,
[CacheTimestampProp]: timestamp || Date.now(),
};
value = `${value}${CacheMetaSplitter}${JSON.stringify(meta)}`;
} }
} else if (value) {
console.error( const cacheObj = {
`${this.context}::prepareValue: keyValue is not object or string`,
value, value,
); parentKeys,
throw new Error( timestamp: timestamp || Date.now(),
`${this.context}::prepareValue: keyValue is not object or string`, };
);
}
return value;
}
async refreshTTL(key: string): Promise<void> { return cacheObj;
log(`${this.context}::refreshTTL: refreshing key ${key}`);
const value = await this.get(key, CacheGetType.TYPE_OBJECT);
if (value) {
const parents = this.getParents(value);
if (parents.length) {
for (const p of parents) {
const childList = await this.get(p, CacheGetType.TYPE_ARRAY);
for (const c of childList) {
const childValue = await this.get(c, CacheGetType.TYPE_OBJECT);
await this.set(c, childValue, { timestamp: Date.now() });
}
}
} else {
await this.set(key, value, { timestamp: Date.now() });
}
}
} }
getParents(value) { getParents(rawValue) {
if (value && typeof value === 'object') { if (rawValue && rawValue.parentKeys) {
if (CacheListProp in value) { return rawValue.parentKeys;
const listsForKey = value[CacheListProp]; } else if (!rawValue) {
if (listsForKey && listsForKey.length) {
return listsForKey;
}
}
} else if (value && typeof value === 'string') {
if (value.includes(CacheListProp)) {
const keyHelper = value.split(CacheListProp);
const listsForKey = keyHelper[1].split(',');
if (listsForKey.length) {
return listsForKey;
}
}
}
return []; return [];
} else {
logger.error(
`${this.context}::getParents: parentKeys not found ${rawValue}`,
);
return [];
}
} }
async destroy(): Promise<boolean> { async destroy(): Promise<boolean> {

6
packages/nocodb/src/utils/globals.ts
Unescape View File

@ -176,12 +176,6 @@ export enum CacheDelDirection {
CHILD_TO_PARENT = 'CHILD_TO_PARENT', CHILD_TO_PARENT = 'CHILD_TO_PARENT',
} }
export const CacheMetaSplitter = '__nc_meta__';
export const CacheListProp = '__nc_list__';
export const CacheTimestampProp = '__nc_timestamp__';
export const GROUPBY_COMPARISON_OPS = <const>[ export const GROUPBY_COMPARISON_OPS = <const>[
// these are used for groupby // these are used for groupby
'gb_eq', 'gb_eq',

Write Preview
Loading…
Cancel
Save