github
/
fair-web
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Fix zip blocks the main thread

pull/6/head
Menci 8 years ago
parent
5917369583
commit
6f09a43063
4 changed files with 76 additions and 63 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      models/file.js
  2. 75
      models/problem.js
  3. 16
      modules/problem.js
  4. 2
      package.json

46
models/file.js
Unescape View File

@ -56,28 +56,43 @@ class File extends Model {
return syzoj.utils.resolvePath(syzoj.config.upload_dir, type, md5); return syzoj.utils.resolvePath(syzoj.config.upload_dir, type, md5);
} }
static async upload(pathOrData, type, noLimit) { // data: Array of { filename: string, data: buffer or string }
static async zipFiles(files) {
let tmp = require('tmp-promise');
let dir = await tmp.dir(), path = require('path'), fs = require('fs-extra');
let filenames = await files.mapAsync(async file => {
let fullPath = path.join(dir.path, file.filename);
await fs.writeFileAsync(fullPath, file.data);
return fullPath;
});
let p7zip = new (require('node-7z')), zipFile = await tmp.tmpName() + '.zip';
console.log(await p7zip.add(zipFile, filenames));
await fs.removeAsync(dir.path);
return zipFile;
}
static async upload(path, type, noLimit) {
let fs = Promise.promisifyAll(require('fs-extra')); let fs = Promise.promisifyAll(require('fs-extra'));
let buf = Buffer.isBuffer(pathOrData) ? pathOrData : await fs.readFileAsync(pathOrData); let buf = await fs.readFileAsync(path);
if (!noLimit && buf.length > syzoj.config.limit.data_size) throw new ErrorMessage('数据包太大。'); if (!noLimit && buf.length > syzoj.config.limit.data_size) throw new ErrorMessage('数据包太大。');
try { try {
let AdmZip = require('adm-zip'); let p7zip = new (require('node-7z'));
let zip = new AdmZip(buf);
this.unzipSize = 0; this.unzipSize = 0;
for (let x of zip.getEntries()) this.unzipSize += x.header.size; await p7zip.list(path).progress(files => {
for (let file of files) this.unzipSize += file.size;
});
} catch (e) { } catch (e) {
this.unzipSize = null; this.unzipSize = null;
} }
let key = syzoj.utils.md5(buf); let key = syzoj.utils.md5(buf);
if (Buffer.isBuffer(pathOrData)) { await fs.moveAsync(path, File.resolvePath(type, key), { overwrite: true });
await fs.writeFileAsync(File.resolvePath(type, key), pathOrData);
} else {
await fs.moveAsync(pathOrData, File.resolvePath(type, key), { overwrite: true });
}
let file = await File.findOne({ where: { md5: key } }); let file = await File.findOne({ where: { md5: key } });
if (!file) { if (!file) {
@ -94,14 +109,11 @@ class File extends Model {
async getUnzipSize() { async getUnzipSize() {
if (this.unzipSize === undefined) { if (this.unzipSize === undefined) {
try { try {
let fs = Promise.promisifyAll(require('fs-extra')); let p7zip = new (require('node-7z'));
let buf = await fs.readFileAsync(this.getPath());
let AdmZip = require('adm-zip');
let zip = new AdmZip(buf);
this.unzipSize = 0; this.unzipSize = 0;
for (let x of zip.getEntries()) this.unzipSize += x.header.size; await p7zip.list(this.getPath()).progress(files => {
for (let file of files) this.unzipSize += file.size;
});
} catch (e) { } catch (e) {
this.unzipSize = null; this.unzipSize = null;
} }

75
models/problem.js
Unescape View File

@ -319,56 +319,63 @@ class Problem extends Model {
} }
async updateTestdata(path, noLimit) { async updateTestdata(path, noLimit) {
let AdmZip = require('adm-zip'); await syzoj.utils.lock(['Problem::Testdata', this.id], async () => {
let zip = new AdmZip(path); let p7zip = new (require('node-7z'));
let unzipSize = 0; let unzipSize = 0;
for (let x of zip.getEntries()) unzipSize += x.header.size; await p7zip.list(path).progress(files => {
if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。'); for (let file of files) unzipSize += file.size;
});
if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
let dir = this.getTestdataPath(); let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra')); let fs = Promise.promisifyAll(require('fs-extra'));
await fs.removeAsync(dir); await fs.removeAsync(dir);
await fs.ensureDirAsync(dir); await fs.ensureDirAsync(dir);
zip.extractAllTo(dir); await p7zip.extract(path, dir);
await fs.moveAsync(path, dir + '.zip', { overwrite: true }); await fs.moveAsync(path, dir + '.zip', { overwrite: true });
});
} }
async uploadTestdataSingleFile(filename, filepath, size, noLimit) { async uploadTestdataSingleFile(filename, filepath, size, noLimit) {
let dir = this.getTestdataPath(); await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path'); let dir = this.getTestdataPath();
await fs.ensureDirAsync(dir); let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.ensureDirAsync(dir);
let oldSize = 0;
let list = await this.listTestdata();
if (list) {
for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
}
if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。'); let oldSize = 0;
let list = await this.listTestdata();
if (list) {
for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
}
if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true }); await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true });
await fs.removeAsync(dir + '.zip'); await fs.removeAsync(dir + '.zip');
});
} }
async deleteTestdataSingleFile(filename) { async deleteTestdataSingleFile(filename) {
let dir = this.getTestdataPath(); await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path'); let dir = this.getTestdataPath();
await fs.removeAsync(path.join(dir, filename)); let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.removeAsync(dir + '.zip'); await fs.removeAsync(path.join(dir, filename));
await fs.removeAsync(dir + '.zip');
});
} }
async makeTestdataZip() { async makeTestdataZip() {
let dir = this.getTestdataPath(); await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。'); let dir = this.getTestdataPath();
if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。');
let AdmZip = require('adm-zip'); let p7zip = new (require('node-7z'));
let zip = new AdmZip();
let list = await this.listTestdata(); let list = await this.listTestdata(), path = require('path');
for (let file of list.files) zip.addLocalFile(require('path').join(dir, file.filename), '', file.filename); await p7zip.add(dir + '.zip', list.files.map(file => path.join(dir, file.filename)));
zip.writeZip(dir + '.zip'); });
} }
async hasSpecialJudge() { async hasSpecialJudge() {

16
modules/problem.js
Unescape View File

@ -568,27 +568,21 @@ app.post('/problem/:id/submit', app.multer.fields([{ name: 'answer', maxCount: 1
let judge_state; let judge_state;
if (problem.type === 'submit-answer') { if (problem.type === 'submit-answer') {
let pathOrData; let File = syzoj.model('file'), path;
if (!req.files['answer']) { if (!req.files['answer']) {
// Submited by editor // Submited by editor
try { try {
let files = JSON.parse(req.body.answer_by_editor); path = await File.zipFiles(JSON.parse(req.body.answer_by_editor));
let AdmZip = require('adm-zip');
let zip = new AdmZip();
for (let file of files) {
zip.addFile(file.filename, file.data);
}
pathOrData = zip.toBuffer();
} catch (e) { } catch (e) {
console.log(e);
throw new ErrorMessage('无法解析提交数据。'); throw new ErrorMessage('无法解析提交数据。');
} }
} else { } else {
if (req.files['answer'][0].size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。'); if (req.files['answer'][0].size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');
pathOrData = req.files['answer'][0].path; path = req.files['answer'][0].path;
} }
let File = syzoj.model('file'); let file = await File.upload(path, 'answer');
let file = await File.upload(pathOrData, 'answer');
let size = await file.getUnzipSize(); let size = await file.getUnzipSize();
if (size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。'); if (size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');

2
package.json
Unescape View File

@ -23,7 +23,6 @@
}, },
"homepage": "https://github.com/syzoj/syzoj#readme", "homepage": "https://github.com/syzoj/syzoj#readme",
"dependencies": { "dependencies": {
"adm-zip": "^0.4.7",
"ansi-to-html": "^0.4.2", "ansi-to-html": "^0.4.2",
"async-lock": "^0.3.9", "async-lock": "^0.3.9",
"body-parser": "^1.15.2", "body-parser": "^1.15.2",
@ -41,6 +40,7 @@
"moment": "^2.15.0", "moment": "^2.15.0",
"multer": "^1.2.0", "multer": "^1.2.0",
"mysql": "^2.11.1", "mysql": "^2.11.1",
"node-7z": "^0.4.0",
"pygmentize-bundled-cached": "^1.1.0", "pygmentize-bundled-cached": "^1.1.0",
"request": "^2.74.0", "request": "^2.74.0",
"request-promise": "^4.1.1", "request-promise": "^4.1.1",

Write Preview
Loading…
Cancel
Save