zhaojunzhe
/
fair-web
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Limit redis cache size of renderer by using a LRU cache

master
Menci 6 years ago
parent
6218e49278
commit
0cb98135d8
5 changed files with 103 additions and 94 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      config-example.json
  2. 64
      libs/renderer.js
  3. 124
      libs/rendererd.js
  4. 1
      package.json
  5. 5
      yarn.lock

3
config-example.json
Unescape View File

@ -126,5 +126,6 @@
"no_cdn": false,
"submissions_page_fast_pagination": false,
"username_regex": "^[a-zA-Z0-9\\-\\_]+$",
"site_for_download": ""
"site_for_download": "",
"renderer_cache_size": "30000"
}

64
libs/renderer.js
Unescape View File

@ -1,32 +1,32 @@
const child_process = require('child_process');
const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis]);
const resolver = {};
let currentId = 0;
rendererd.on('message', msg => {
resolver[msg.id](msg.result);
delete resolver[msg.id];
});
exports.markdown = (markdownCode, callback) => {
resolver[++currentId] = callback;
rendererd.send({
id: currentId,
type: 'markdown',
source: markdownCode
});
}
exports.highlight = (code, lang, callback) => {
resolver[++currentId] = callback;
rendererd.send({
id: currentId,
type: 'highlight',
source: {
code,
lang
}
});
}
const child_process = require('child_process');
const rendererd = child_process.fork(__dirname + '/rendererd', [syzoj.config.redis, parseInt(syzoj.config.renderer_cache_size)]);
const resolver = {};
let currentId = 0;
rendererd.on('message', msg => {
resolver[msg.id](msg.result);
delete resolver[msg.id];
});
exports.markdown = (markdownCode, callback) => {
resolver[++currentId] = callback;
rendererd.send({
id: currentId,
type: 'markdown',
source: markdownCode
});
}
exports.highlight = (code, lang, callback) => {
resolver[++currentId] = callback;
rendererd.send({
id: currentId,
type: 'highlight',
source: {
code,
lang
}
});
}

124
libs/rendererd.js
Unescape View File

@ -1,61 +1,63 @@
const renderer = require('syzoj-renderer');
const XSS = require('xss');
const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList);
delete xssWhiteList.audio;
delete xssWhiteList.video;
for (const tag in xssWhiteList) {
xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']);
}
const xss = new XSS.FilterXSS({
whiteList: xssWhiteList,
stripIgnoreTag: true,
onTagAttr: (tag, name, value, isWhiteAttr) => {
if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) {
return name + '="' + XSS.escapeAttrValue(value) + '"';
}
}
});
const Redis = require('redis');
const util = require('util');
const redis = Redis.createClient(process.argv[2]);
const redisCache = {
get: util.promisify(redis.get).bind(redis),
set: util.promisify(redis.set).bind(redis)
};
async function highlight(code, lang) {
return await renderer.highlight(code, lang, redisCache, {
wrapper: null
});
}
async function markdown(markdownCode) {
function filter(html) {
html = xss.process(html);
if (html) {
html = `<div style="position: relative; overflow: hidden; ">${html}</div>`;
}
return html;
};
return await renderer.markdown(markdownCode, redisCache, filter);
}
process.on('message', async msg => {
if (msg.type === 'markdown') {
process.send({
id: msg.id,
result: await markdown(msg.source)
});
} else if (msg.type === 'highlight') {
process.send({
id: msg.id,
result: await highlight(msg.source.code, msg.source.lang)
});
}
});
process.on('disconnect', () => process.exit());
const renderer = require('syzoj-renderer');
const XSS = require('xss');
const xssWhiteList = Object.assign({}, require('xss/lib/default').whiteList);
delete xssWhiteList.audio;
delete xssWhiteList.video;
for (const tag in xssWhiteList) {
xssWhiteList[tag] = xssWhiteList[tag].concat(['style', 'class']);
}
const xss = new XSS.FilterXSS({
whiteList: xssWhiteList,
stripIgnoreTag: true,
onTagAttr: (tag, name, value, isWhiteAttr) => {
if (tag.toLowerCase() === 'img' && name.toLowerCase() === 'src' && value.startsWith('data:image/')) {
return name + '="' + XSS.escapeAttrValue(value) + '"';
}
}
});
const Redis = require('redis');
const RedisLRU = require('redis-lru');
const util = require('util');
const redis = Redis.createClient(process.argv[2]);
const redisLru = RedisLRU(redis, parseInt(process.argv[3]));
const redisCache = {
get: redisLru.get.bind(redisLru),
set: redisLru.set.bind(redisLru)
};
async function highlight(code, lang) {
return await renderer.highlight(code, lang, redisCache, {
wrapper: null
});
}
async function markdown(markdownCode) {
function filter(html) {
html = xss.process(html);
if (html) {
html = `<div style="position: relative; overflow: hidden; ">${html}</div>`;
}
return html;
};
return await renderer.markdown(markdownCode, redisCache, filter);
}
process.on('message', async msg => {
if (msg.type === 'markdown') {
process.send({
id: msg.id,
result: await markdown(msg.source)
});
} else if (msg.type === 'highlight') {
process.send({
id: msg.id,
result: await highlight(msg.source.code, msg.source.lang)
});
}
});
process.on('disconnect', () => process.exit());

1
package.json
Unescape View File

@ -59,6 +59,7 @@
"object-hash": "^1.3.1",
"randomstring": "^1.1.5",
"redis": "^2.8.0",
"redis-lru": "^0.6.0",
"reflect-metadata": "^0.1.13",
"request": "^2.74.0",
"request-promise": "^4.2.4",

5
yarn.lock
Unescape View File

@ -2881,6 +2881,11 @@ redis-commands@^1.2.0:
resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.4.0.tgz#52f9cf99153efcce56a8f86af986bd04e988602f"
integrity sha512-cu8EF+MtkwI4DLIT0x9P8qNTLFhQD4jLfxLR0cCNkeGzs87FN6879JOJwNQR/1zD7aSYNbU0hgsV9zGY71Itvw==
redis-lru@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/redis-lru/-/redis-lru-0.6.0.tgz#2a820a99011fe1a16a7ad2d67980def9d3336c7a"
integrity sha512-nDH+EHcUXp+cJdnkq8Hb+99kr82EDmAiFY40Bb02nB00A4N/dUqw3SCIJSgH8SUP2uee9EFijbMlf5WSul22uw==
redis-parser@^2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"

Write Preview
Loading…
Cancel
Save