feat: wrap all cache with metadata

packages/nocodb/src/cache/CacheMgr.ts

@@ -1,14 +1,10 @@
 import debug from 'debug';
+import { Logger } from '@nestjs/common';
 import type IORedis from 'ioredis';
-import {
+import { CacheDelDirection, CacheGetType } from '~/utils/globals';
-  CacheDelDirection,
-  CacheGetType,
-  CacheListProp,
-  CacheMetaSplitter,
-  CacheTimestampProp,
-} from '~/utils/globals';
 
 const log = debug('nc:cache');
+const logger = new Logger('CacheMgr');
 export default abstract class CacheMgr {
   client: IORedis;
   prefix: string;
@@ -41,38 +37,42 @@ export default abstract class CacheMgr {
   }
 
   // @ts-ignore
-  async get(key: string, type: string): Promise<any> {
+  private async getRaw(key: string, type?: string): Promise<any> {
-    log(`${this.context}::get: getting key ${key} with type ${type}`);
+    log(`${this.context}::getRaw: getting key ${key} with type ${type}`);
     if (type === CacheGetType.TYPE_ARRAY) {
       return this.client.smembers(key);
-    } else if (type === CacheGetType.TYPE_OBJECT) {
+    } else {
       const res = await this.client.get(key);
-      try {
+      if (res) {
-        const o = JSON.parse(res);
+        try {
-        if (typeof o === 'object') {
+          const o = JSON.parse(res);
-          if (
+          if (typeof o === 'object') {
-            o &&
+            if (
-            Object.keys(o).length === 0 &&
+              o &&
-            Object.getPrototypeOf(o) === Object.prototype
+              Object.keys(o).length === 0 &&
-          ) {
+              Object.getPrototypeOf(o) === Object.prototype
-            log(`${this.context}::get: object is empty!`);
+            ) {
+              log(`${this.context}::get: object is empty!`);
+            }
+            return Promise.resolve(o);
           }
-          return Promise.resolve(o);
+        } catch (e) {
-        }
+          logger.error(`Bad value stored for key ${key} : ${res}`);
-      } catch (e) {}
+          return Promise.resolve(res);
-      const valueHelper = res.split(CacheMetaSplitter);
-      return Promise.resolve(valueHelper[0]);
-    } else if (type === CacheGetType.TYPE_STRING) {
-      return this.client.get(key).then((res) => {
-        if (!res) {
-          return res;
         }
-        const valueHelper = res.split(CacheMetaSplitter);
+      }
-        return valueHelper[0];
+      return Promise.resolve(res);
-      });
     }
-    log(`Invalid CacheGetType: ${type}`);
+  }
-    return Promise.resolve(false);
+
+  // @ts-ignore
+  async get(key: string, type: string): Promise<any> {
+    return this.getRaw(key, type).then((res) => {
+      if (res && res.value) {
+        return res.value;
+      }
+      return res;
+    });
   }
 
   // @ts-ignore
@@ -80,7 +80,9 @@ export default abstract class CacheMgr {
     key: string,
     value: any,
     options: {
+      // when we prepare beforehand, we don't need to prepare again
       skipPrepare?: boolean;
+      // timestamp for the value, if not provided, it will be set to current time
       timestamp?: number;
     } = {
       skipPrepare: false,
@@ -91,38 +93,25 @@ export default abstract class CacheMgr {
     if (typeof value !== 'undefined' && value) {
       log(`${this.context}::set: setting key ${key} with value ${value}`);
 
-      if (typeof value === 'object') {
+      if (Array.isArray(value) && value.length) {
-        if (Array.isArray(value) && value.length) {
+        return this.client.sadd(key, value);
-          return this.client.sadd(key, value);
-        }
-
-        if (!skipPrepare) {
-          // try to get old key value
-          const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT);
-          // prepare new key value
-          value = this.prepareValue(value, {
-            parentKeys: this.getParents(keyValue),
-            timestamp,
-          });
-        }
-
-        return this.client.set(
-          key,
-          JSON.stringify(value, this.getCircularReplacer()),
-        );
       }
 
       if (!skipPrepare) {
         // try to get old key value
-        const keyValue = await this.get(key, CacheGetType.TYPE_OBJECT);
+        const keyValue = await this.getRaw(key);
         // prepare new key value
-        value = this.prepareValue(value.toString(), {
+        value = this.prepareValue({
+          value,
           parentKeys: this.getParents(keyValue),
           timestamp,
         });
       }
 
-      return this.client.set(key, value);
+      return this.client.set(
+        key,
+        JSON.stringify(value, this.getCircularReplacer()),
+      );
     } else {
       log(`${this.context}::set: value is empty for ${key}. Skipping ...`);
       return Promise.resolve(true);
@@ -130,23 +119,47 @@ export default abstract class CacheMgr {
   }
 
   // @ts-ignore
-  async setExpiring(key: string, value: any, seconds: number): Promise<any> {
+  async setExpiring(
+    key: string,
+    value: any,
+    seconds: number,
+    options: {
+      // when we prepare beforehand, we don't need to prepare again
+      skipPrepare?: boolean;
+      // timestamp for the value, if not provided, it will be set to current time
+      timestamp?: number;
+    } = {
+      skipPrepare: false,
+    },
+  ): Promise<any> {
+    const { skipPrepare, timestamp } = options;
+
     if (typeof value !== 'undefined' && value) {
       log(
-        `${this.context}::setExpiring: setting key ${key} with value ${value} for ${seconds} seconds`,
+        `${this.context}::setExpiring: setting key ${key} with value ${value}`,
       );
-      if (typeof value === 'object') {
+
-        if (Array.isArray(value) && value.length) {
+      if (Array.isArray(value) && value.length) {
-          return this.client.sadd(key, value);
+        return this.client.sadd(key, value);
-        }
-        return this.client.set(
-          key,
-          JSON.stringify(value, this.getCircularReplacer()),
-          'EX',
-          seconds,
-        );
       }
-      return this.client.set(key, value, 'EX', seconds);
+
+      if (!skipPrepare) {
+        // try to get old key value
+        const keyValue = await this.getRaw(key);
+        // prepare new key value
+        value = this.prepareValue({
+          value,
+          parentKeys: this.getParents(keyValue),
+          timestamp,
+        });
+      }
+
+      return this.client.set(
+        key,
+        JSON.stringify(value, this.getCircularReplacer()),
+        'EX',
+        seconds,
+      );
     } else {
       log(`${this.context}::set: value is empty for ${key}. Skipping ...`);
       return Promise.resolve(true);
@@ -189,7 +202,7 @@ export default abstract class CacheMgr {
 
     if (values.some((v) => v === null)) {
       // FALLBACK: a key is missing from list, this should never happen
-      console.error(`${this.context}::getList: missing value for ${key}`);
+      logger.error(`${this.context}::getList: missing value for ${key}`);
       const allParents = [];
       // get all parents from children
       values.forEach((v) => {
@@ -214,7 +227,7 @@ export default abstract class CacheMgr {
         try {
           const o = JSON.parse(res);
           if (typeof o === 'object') {
-            return o;
+            return o.value;
           }
         } catch (e) {
           return res;
@@ -260,30 +273,29 @@ export default abstract class CacheMgr {
       }
       log(`${this.context}::setList: get key ${getKey}`);
       // get key
-      let value = await this.get(getKey, CacheGetType.TYPE_OBJECT);
+      let value = await this.getRaw(getKey, CacheGetType.TYPE_OBJECT);
       if (value) {
         log(`${this.context}::setList: preparing key ${getKey}`);
         // prepare key
-        value = this.prepareValue(o, {
+        value = this.prepareValue({
-          parentKeys: this.getParents(value).concat(listKey),
+          value: o,
+          parentKeys: this.getParents(value),
+          newKey: listKey,
           timestamp,
         });
       } else {
-        value = this.prepareValue(o, {
+        value = this.prepareValue({
+          value: o,
           parentKeys: [listKey],
           timestamp,
         });
       }
       // set key
       log(`${this.context}::setList: setting key ${getKey}`);
-      await this.set(
+      await this.set(getKey, value, {
-        getKey,
+        skipPrepare: true,
-        JSON.stringify(value, this.getCircularReplacer()),
+        timestamp,
-        {
+      });
-          skipPrepare: true,
-          timestamp,
-        },
-      );
       // push key to list
       listOfGetKeys.push(getKey);
     }
@@ -361,7 +373,7 @@ export default abstract class CacheMgr {
     log(`${this.context}::appendToList: preparing key ${key}`);
     if (!value) {
       // FALLBACK: this is to get rid of all keys that would be effected by this (should never happen)
-      console.error(`${this.context}::appendToList: value is empty for ${key}`);
+      logger.error(`${this.context}::appendToList: value is empty for ${key}`);
       const allParents = [];
       // get all children
       const listValues = await this.getList(scope, subListKeys);
@@ -380,115 +392,54 @@ export default abstract class CacheMgr {
       return false;
     }
     // prepare Get Key
-    const preparedValue = this.prepareValue(value, {
+    const preparedValue = this.prepareValue({
-      parentKeys: this.getParents(value).concat(listKey),
+      value,
+      parentKeys: this.getParents(value),
+      newKey: listKey,
     });
     // set Get Key
     log(`${this.context}::appendToList: setting key ${key}`);
-    await this.set(
+    await this.set(key, preparedValue, {
-      key,
+      skipPrepare: true,
-      JSON.stringify(preparedValue, this.getCircularReplacer()),
+    });
-      {
-        skipPrepare: true,
-      },
-    );
 
     list.push(key);
     return this.set(listKey, list);
   }
 
-  prepareValue(
+  // wrap value with metadata
-    value,
+  prepareValue(args: {
-    options: {
+    value: any;
-      parentKeys: string[];
+    parentKeys: string[];
-      timestamp?: number;
+    newKey?: string;
-    },
+    timestamp?: number;
-  ) {
+  }) {
-    const { parentKeys, timestamp } = options;
+    const { value, parentKeys, newKey, timestamp } = args;
-
+
-    if (value && typeof value === 'object') {
+    if (newKey && !parentKeys.includes(newKey)) {
-      value[CacheListProp] = parentKeys;
+      parentKeys.push(newKey);
-
-      if (timestamp) {
-        value[CacheTimestampProp] = timestamp;
-      } else {
-        value[CacheTimestampProp] = Date.now();
-      }
-    } else if (value && typeof value === 'string') {
-      const metaHelper = value.split(CacheMetaSplitter);
-      if (metaHelper.length > 1) {
-        const keyVal = metaHelper[0];
-        const keyMeta = metaHelper[1];
-        try {
-          const meta = JSON.parse(keyMeta);
-          meta[CacheListProp] = parentKeys;
-          meta[CacheTimestampProp] = timestamp || Date.now();
-          value = `${keyVal}${CacheMetaSplitter}${JSON.stringify(meta)}`;
-        } catch (e) {
-          console.error(
-            `${this.context}::prepareValue: keyValue meta is not JSON`,
-            keyMeta,
-          );
-          throw new Error(
-            `${this.context}::prepareValue: keyValue meta is not JSON`,
-          );
-        }
-      } else {
-        const meta = {
-          [CacheListProp]: parentKeys,
-          [CacheTimestampProp]: timestamp || Date.now(),
-        };
-        value = `${value}${CacheMetaSplitter}${JSON.stringify(meta)}`;
-      }
-    } else if (value) {
-      console.error(
-        `${this.context}::prepareValue: keyValue is not object or string`,
-        value,
-      );
-      throw new Error(
-        `${this.context}::prepareValue: keyValue is not object or string`,
-      );
     }
-    return value;
-  }
 
-  async refreshTTL(key: string): Promise<void> {
+    const cacheObj = {
-    log(`${this.context}::refreshTTL: refreshing key ${key}`);
+      value,
-    const value = await this.get(key, CacheGetType.TYPE_OBJECT);
+      parentKeys,
-    if (value) {
+      timestamp: timestamp || Date.now(),
-      const parents = this.getParents(value);
+    };
-      if (parents.length) {
+
-        for (const p of parents) {
+    return cacheObj;
-          const childList = await this.get(p, CacheGetType.TYPE_ARRAY);
-          for (const c of childList) {
-            const childValue = await this.get(c, CacheGetType.TYPE_OBJECT);
-            await this.set(c, childValue, { timestamp: Date.now() });
-          }
-        }
-      } else {
-        await this.set(key, value, { timestamp: Date.now() });
-      }
-    }
   }
 
-  getParents(value) {
+  getParents(rawValue) {
-    if (value && typeof value === 'object') {
+    if (rawValue && rawValue.parentKeys) {
-      if (CacheListProp in value) {
+      return rawValue.parentKeys;
-        const listsForKey = value[CacheListProp];
+    } else if (!rawValue) {
-        if (listsForKey && listsForKey.length) {
+      return [];
-          return listsForKey;
+    } else {
-        }
+      logger.error(
-      }
+        `${this.context}::getParents: parentKeys not found ${rawValue}`,
-    } else if (value && typeof value === 'string') {
+      );
-      if (value.includes(CacheListProp)) {
+      return [];
-        const keyHelper = value.split(CacheListProp);
-        const listsForKey = keyHelper[1].split(',');
-        if (listsForKey.length) {
-          return listsForKey;
-        }
-      }
     }
-    return [];
   }
 
   async destroy(): Promise<boolean> {

packages/nocodb/src/utils/globals.ts

@@ -176,12 +176,6 @@ export enum CacheDelDirection {
   CHILD_TO_PARENT = 'CHILD_TO_PARENT',
 }
 
-export const CacheMetaSplitter = '__nc_meta__';
-
-export const CacheListProp = '__nc_list__';
-
-export const CacheTimestampProp = '__nc_timestamp__';
-
 export const GROUPBY_COMPARISON_OPS = <const>[
   // these are used for groupby
   'gb_eq',