Limit redis cache size of renderer by using a LRU cache

config-example.json

@@ -126,5 +126,6 @@
   "no_cdn": false,
   "submissions_page_fast_pagination": false,
   "username_regex": "^[a-zA-Z0-9\\-\\_]+$",
-  "site_for_download": ""
+  "site_for_download": "",
+  "renderer_cache_size": "30000"
 }

libs/renderer.js

@@ -1,32 +1,32 @@
-const child_process = require('child_process');
+const child_process = require('child_process');
-
+
-const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis]);
+const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis, parseInt(syzoj.config.renderer_cache_size)]);
-
+
-const resolver = {};
+const resolver = {};
-let currentId = 0;
+let currentId = 0;
-
+
-rendererd.on('message', msg => {
+rendererd.on('message', msg => {
-  resolver[msg.id](msg.result);
+  resolver[msg.id](msg.result);
-  delete resolver[msg.id];
+  delete resolver[msg.id];
-});
+});
-
+
-exports.markdown = (markdownCode, callback) => {
+exports.markdown = (markdownCode, callback) => {
-  resolver[++currentId] = callback;
+  resolver[++currentId] = callback;
-  rendererd.send({
+  rendererd.send({
-    id: currentId,
+    id: currentId,
-    type: 'markdown',
+    type: 'markdown',
-    source: markdownCode
+    source: markdownCode
-  });
+  });
-}
+}
-
+
-exports.highlight = (code, lang, callback) => {
+exports.highlight = (code, lang, callback) => {
-  resolver[++currentId] = callback;
+  resolver[++currentId] = callback;
-  rendererd.send({
+  rendererd.send({
-    id: currentId,
+    id: currentId,
-    type: 'highlight',
+    type: 'highlight',
-    source: {
+    source: {
-      code,
+      code,
-      lang
+      lang
-    }
+    }
-  });
+  });
-}
+}

libs/rendererd.js

@@ -1,61 +1,63 @@
-const renderer = require('syzoj-renderer');
+const renderer = require('syzoj-renderer');
-const XSS = require('xss');
+const XSS = require('xss');
-const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList);
+const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList);
-delete xssWhiteList.audio;
+delete xssWhiteList.audio;
-delete xssWhiteList.video;
+delete xssWhiteList.video;
-
+
-for (const tag in xssWhiteList) {
+for (const tag in xssWhiteList) {
-  xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']);
+  xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']);
-}
+}
-
+
-const xss = new XSS.FilterXSS({
+const xss = new XSS.FilterXSS({
-  whiteList: xssWhiteList,
+  whiteList: xssWhiteList,
-  stripIgnoreTag: true,
+  stripIgnoreTag: true,
-  onTagAttr: (tag, name, value, isWhiteAttr) => {
+  onTagAttr: (tag, name, value, isWhiteAttr) => {
-    if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) {
+    if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) {
-      return name + '="' + XSS.escapeAttrValue(value) + '"';
+      return name + '="' + XSS.escapeAttrValue(value) + '"';
-    }
+    }
-  }
+  }
-});
+});
-
+
-const Redis = require('redis');
+const Redis = require('redis');
-const util = require('util');
+const RedisLRU = require('redis-lru');
-const redis = Redis.createClient(process.argv[2]);
+const util = require('util');
-const redisCache = {
+const redis = Redis.createClient(process.argv[2]);
-  get: util.promisify(redis.get).bind(redis),
+const redisLru = RedisLRU(redis, parseInt(process.argv[3]));
-  set: util.promisify(redis.set).bind(redis)
+const redisCache = {
-};
+  get: redisLru.get.bind(redisLru),
-
+  set: redisLru.set.bind(redisLru)
-async function highlight(code, lang) {
+};
-  return await renderer.highlight(code, lang, redisCache, {
+
-    wrapper: null
+async function highlight(code, lang) {
-  });
+  return await renderer.highlight(code, lang, redisCache, {
-}
+    wrapper: null
-
+  });
-async function markdown(markdownCode) {
+}
-  function filter(html) {
+
-    html = xss.process(html);
+async function markdown(markdownCode) {
-    if (html) {
+  function filter(html) {
-      html = `<div style="position: relative; overflow: hidden; ">${html}</div>`;
+    html = xss.process(html);
-    }
+    if (html) {
-    return html;
+      html = `<div style="position: relative; overflow: hidden; ">${html}</div>`;
-  };
+    }
-
+    return html;
-  return await renderer.markdown(markdownCode, redisCache, filter);
+  };
-}
+
-
+  return await renderer.markdown(markdownCode, redisCache, filter);
-process.on('message', async msg => {
+}
-  if (msg.type === 'markdown') {
+
-    process.send({
+process.on('message', async msg => {
-      id: msg.id,
+  if (msg.type === 'markdown') {
-      result: await markdown(msg.source)
+    process.send({
-    });
+      id: msg.id,
-  } else if (msg.type === 'highlight') {
+      result: await markdown(msg.source)
-    process.send({
+    });
-      id: msg.id,
+  } else if (msg.type === 'highlight') {
-      result: await highlight(msg.source.code, msg.source.lang)
+    process.send({
-    });
+      id: msg.id,
-  }
+      result: await highlight(msg.source.code, msg.source.lang)
-});
+    });
-
+  }
-process.on('disconnect', () => process.exit());
+});
+
+process.on('disconnect', () => process.exit());

package.json

@@ -59,6 +59,7 @@
     "object-hash": "^1.3.1",
     "randomstring": "^1.1.5",
     "redis": "^2.8.0",
+    "redis-lru": "^0.6.0",
     "reflect-metadata": "^0.1.13",
     "request": "^2.74.0",
     "request-promise": "^4.2.4",

yarn.lock

@@ -2881,6 +2881,11 @@ redis-commands@^1.2.0:
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f"
   integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw==
 
+redis-lru@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/redis-lru/-/redis-lru-0.6.0.tgz#2a820a99011fe1a16a7ad2d67980def9d3336c7a"
+  integrity sha512-nDH+EHcUXp+cJdnkq8Hb+99kr82EDmAiFY40Bb02nB00A4N/dUqw3SCIJSgH8SUP2uee9EFijbMlf5WSul22uw==
+
 redis-parser@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"