From 0cb98135d810345f615492731dcd294b37bfd177 Mon Sep 17 00:00:00 2001 From: Menci Date: Mon, 3 Jun 2019 18:46:53 +0800 Subject: [PATCH] Limit redis cache size of renderer by using a LRU cache --- config-example.json | 3 +- libs/renderer.js | 64 +++++++++++------------ libs/rendererd.js | 124 ++++++++++++++++++++++---------------------- package.json | 1 + yarn.lock | 5 ++ 5 files changed, 103 insertions(+), 94 deletions(-) diff --git a/config-example.json b/config-example.json index 17bbd3a..1091a17 100644 --- a/config-example.json +++ b/config-example.json @@ -126,5 +126,6 @@ "no_cdn": false, "submissions_page_fast_pagination": false, "username_regex": "^[a-zA-Z0-9\\-\\_]+$", - "site_for_download": "" + "site_for_download": "", + "renderer_cache_size": "30000" } diff --git a/libs/renderer.js b/libs/renderer.js index eacbec6..b40d1e8 100644 --- a/libs/renderer.js +++ b/libs/renderer.js @@ -1,32 +1,32 @@ -const child_process = require('child_process'); - -const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis]); - -const resolver = {}; -let currentId = 0; - -rendererd.on('message', msg => { - resolver[msg.id](msg.result); - delete resolver[msg.id]; -}); - -exports.markdown = (markdownCode, callback) => { - resolver[++currentId] = callback; - rendererd.send({ - id: currentId, - type: 'markdown', - source: markdownCode - }); -} - -exports.highlight = (code, lang, callback) => { - resolver[++currentId] = callback; - rendererd.send({ - id: currentId, - type: 'highlight', - source: { - code, - lang - } - }); -} +const child_process = require('child_process'); + +const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis, parseInt(syzoj.config.renderer_cache_size)]); + +const resolver = {}; +let currentId = 0; + +rendererd.on('message', msg => { + resolver[msg.id](msg.result); + delete resolver[msg.id]; +}); + +exports.markdown = (markdownCode, callback) => { + resolver[++currentId] = callback; + rendererd.send({ + id: currentId, + type: 'markdown', + source: markdownCode + }); +} + +exports.highlight = (code, lang, callback) => { + resolver[++currentId] = callback; + rendererd.send({ + id: currentId, + type: 'highlight', + source: { + code, + lang + } + }); +} diff --git a/libs/rendererd.js b/libs/rendererd.js index 49d5aa4..1b6f535 100644 --- a/libs/rendererd.js +++ b/libs/rendererd.js @@ -1,61 +1,63 @@ -const renderer = require('syzoj-renderer'); -const XSS = require('xss'); -const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList); -delete xssWhiteList.audio; -delete xssWhiteList.video; - -for (const tag in xssWhiteList) { - xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']); -} - -const xss = new XSS.FilterXSS({ - whiteList: xssWhiteList, - stripIgnoreTag: true, - onTagAttr: (tag, name, value, isWhiteAttr) => { - if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) { - return name + '="' + XSS.escapeAttrValue(value) + '"'; - } - } -}); - -const Redis = require('redis'); -const util = require('util'); -const redis = Redis.createClient(process.argv[2]); -const redisCache = { - get: util.promisify(redis.get).bind(redis), - set: util.promisify(redis.set).bind(redis) -}; - -async function highlight(code, lang) { - return await renderer.highlight(code, lang, redisCache, { - wrapper: null - }); -} - -async function markdown(markdownCode) { - function filter(html) { - html = xss.process(html); - if (html) { - html = `
${html}
`; - } - return html; - }; - - return await renderer.markdown(markdownCode, redisCache, filter); -} - -process.on('message', async msg => { - if (msg.type === 'markdown') { - process.send({ - id: msg.id, - result: await markdown(msg.source) - }); - } else if (msg.type === 'highlight') { - process.send({ - id: msg.id, - result: await highlight(msg.source.code, msg.source.lang) - }); - } -}); - -process.on('disconnect', () => process.exit()); +const renderer = require('syzoj-renderer'); +const XSS = require('xss'); +const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList); +delete xssWhiteList.audio; +delete xssWhiteList.video; + +for (const tag in xssWhiteList) { + xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']); +} + +const xss = new XSS.FilterXSS({ + whiteList: xssWhiteList, + stripIgnoreTag: true, + onTagAttr: (tag, name, value, isWhiteAttr) => { + if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) { + return name + '="' + XSS.escapeAttrValue(value) + '"'; + } + } +}); + +const Redis = require('redis'); +const RedisLRU = require('redis-lru'); +const util = require('util'); +const redis = Redis.createClient(process.argv[2]); +const redisLru = RedisLRU(redis, parseInt(process.argv[3])); +const redisCache = { + get: redisLru.get.bind(redisLru), + set: redisLru.set.bind(redisLru) +}; + +async function highlight(code, lang) { + return await renderer.highlight(code, lang, redisCache, { + wrapper: null + }); +} + +async function markdown(markdownCode) { + function filter(html) { + html = xss.process(html); + if (html) { + html = `
${html}
`; + } + return html; + }; + + return await renderer.markdown(markdownCode, redisCache, filter); +} + +process.on('message', async msg => { + if (msg.type === 'markdown') { + process.send({ + id: msg.id, + result: await markdown(msg.source) + }); + } else if (msg.type === 'highlight') { + process.send({ + id: msg.id, + result: await highlight(msg.source.code, msg.source.lang) + }); + } +}); + +process.on('disconnect', () => process.exit()); diff --git a/package.json b/package.json index 933f3eb..66bdf84 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "object-hash": "^1.3.1", "randomstring": "^1.1.5", "redis": "^2.8.0", + "redis-lru": "^0.6.0", "reflect-metadata": "^0.1.13", "request": "^2.74.0", "request-promise": "^4.2.4", diff --git a/yarn.lock b/yarn.lock index bb257ba..06741f8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2881,6 +2881,11 @@ redis-commands@^1.2.0: resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f" integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw== +redis-lru@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/redis-lru/-/redis-lru-0.6.0.tgz#2a820a99011fe1a16a7ad2d67980def9d3336c7a" + integrity sha512-nDH+EHcUXp+cJdnkq8Hb+99kr82EDmAiFY40Bb02nB00A4N/dUqw3SCIJSgH8SUP2uee9EFijbMlf5WSul22uw== + redis-parser@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"