Browse Source

Fix zip blocks the main thread

master
Menci 8 years ago
parent
commit
6f09a43063
  1. 46
      models/file.js
  2. 75
      models/problem.js
  3. 16
      modules/problem.js
  4. 2
      package.json

46
models/file.js

@ -56,28 +56,43 @@ class File extends Model {
return syzoj.utils.resolvePath(syzoj.config.upload_dir, type, md5);
}
static async upload(pathOrData, type, noLimit) {
// data: Array of { filename: string, data: buffer or string }
static async zipFiles(files) {
let tmp = require('tmp-promise');
let dir = await tmp.dir(), path = require('path'), fs = require('fs-extra');
let filenames = await files.mapAsync(async file => {
let fullPath = path.join(dir.path, file.filename);
await fs.writeFileAsync(fullPath, file.data);
return fullPath;
});
let p7zip = new (require('node-7z')), zipFile = await tmp.tmpName() + '.zip';
console.log(await p7zip.add(zipFile, filenames));
await fs.removeAsync(dir.path);
return zipFile;
}
static async upload(path, type, noLimit) {
let fs = Promise.promisifyAll(require('fs-extra'));
let buf = Buffer.isBuffer(pathOrData) ? pathOrData : await fs.readFileAsync(pathOrData);
let buf = await fs.readFileAsync(path);
if (!noLimit && buf.length > syzoj.config.limit.data_size) throw new ErrorMessage('数据包太大。');
try {
let AdmZip = require('adm-zip');
let zip = new AdmZip(buf);
let p7zip = new (require('node-7z'));
this.unzipSize = 0;
for (let x of zip.getEntries()) this.unzipSize += x.header.size;
await p7zip.list(path).progress(files => {
for (let file of files) this.unzipSize += file.size;
});
} catch (e) {
this.unzipSize = null;
}
let key = syzoj.utils.md5(buf);
if (Buffer.isBuffer(pathOrData)) {
await fs.writeFileAsync(File.resolvePath(type, key), pathOrData);
} else {
await fs.moveAsync(pathOrData, File.resolvePath(type, key), { overwrite: true });
}
await fs.moveAsync(path, File.resolvePath(type, key), { overwrite: true });
let file = await File.findOne({ where: { md5: key } });
if (!file) {
@ -94,14 +109,11 @@ class File extends Model {
async getUnzipSize() {
if (this.unzipSize === undefined) {
try {
let fs = Promise.promisifyAll(require('fs-extra'));
let buf = await fs.readFileAsync(this.getPath());
let AdmZip = require('adm-zip');
let zip = new AdmZip(buf);
let p7zip = new (require('node-7z'));
this.unzipSize = 0;
for (let x of zip.getEntries()) this.unzipSize += x.header.size;
await p7zip.list(this.getPath()).progress(files => {
for (let file of files) this.unzipSize += file.size;
});
} catch (e) {
this.unzipSize = null;
}

75
models/problem.js

@ -319,56 +319,63 @@ class Problem extends Model {
}
async updateTestdata(path, noLimit) {
let AdmZip = require('adm-zip');
let zip = new AdmZip(path);
await syzoj.utils.lock(['Problem::Testdata', this.id], async () => {
let p7zip = new (require('node-7z'));
let unzipSize = 0;
for (let x of zip.getEntries()) unzipSize += x.header.size;
if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
let unzipSize = 0;
await p7zip.list(path).progress(files => {
for (let file of files) unzipSize += file.size;
});
if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra'));
await fs.removeAsync(dir);
await fs.ensureDirAsync(dir);
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra'));
await fs.removeAsync(dir);
await fs.ensureDirAsync(dir);
zip.extractAllTo(dir);
await fs.moveAsync(path, dir + '.zip', { overwrite: true });
await p7zip.extract(path, dir);
await fs.moveAsync(path, dir + '.zip', { overwrite: true });
});
}
async uploadTestdataSingleFile(filename, filepath, size, noLimit) {
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.ensureDirAsync(dir);
let oldSize = 0;
let list = await this.listTestdata();
if (list) {
for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
}
await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.ensureDirAsync(dir);
if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
let oldSize = 0;
let list = await this.listTestdata();
if (list) {
for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
}
if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true });
await fs.removeAsync(dir + '.zip');
await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true });
await fs.removeAsync(dir + '.zip');
});
}
async deleteTestdataSingleFile(filename) {
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.removeAsync(path.join(dir, filename));
await fs.removeAsync(dir + '.zip');
await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
let dir = this.getTestdataPath();
let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
await fs.removeAsync(path.join(dir, filename));
await fs.removeAsync(dir + '.zip');
});
}
async makeTestdataZip() {
let dir = this.getTestdataPath();
if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。');
await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
let dir = this.getTestdataPath();
if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。');
let AdmZip = require('adm-zip');
let zip = new AdmZip();
let p7zip = new (require('node-7z'));
let list = await this.listTestdata();
for (let file of list.files) zip.addLocalFile(require('path').join(dir, file.filename), '', file.filename);
zip.writeZip(dir + '.zip');
let list = await this.listTestdata(), path = require('path');
await p7zip.add(dir + '.zip', list.files.map(file => path.join(dir, file.filename)));
});
}
async hasSpecialJudge() {

16
modules/problem.js

@ -568,27 +568,21 @@ app.post('/problem/:id/submit', app.multer.fields([{ name: 'answer', maxCount: 1
let judge_state;
if (problem.type === 'submit-answer') {
let pathOrData;
let File = syzoj.model('file'), path;
if (!req.files['answer']) {
// Submited by editor
try {
let files = JSON.parse(req.body.answer_by_editor);
let AdmZip = require('adm-zip');
let zip = new AdmZip();
for (let file of files) {
zip.addFile(file.filename, file.data);
}
pathOrData = zip.toBuffer();
path = await File.zipFiles(JSON.parse(req.body.answer_by_editor));
} catch (e) {
console.log(e);
throw new ErrorMessage('无法解析提交数据。');
}
} else {
if (req.files['answer'][0].size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');
pathOrData = req.files['answer'][0].path;
path = req.files['answer'][0].path;
}
let File = syzoj.model('file');
let file = await File.upload(pathOrData, 'answer');
let file = await File.upload(path, 'answer');
let size = await file.getUnzipSize();
if (size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');

2
package.json

@ -23,7 +23,6 @@
},
"homepage": "https://github.com/syzoj/syzoj#readme",
"dependencies": {
"adm-zip": "^0.4.7",
"ansi-to-html": "^0.4.2",
"async-lock": "^0.3.9",
"body-parser": "^1.15.2",
@ -41,6 +40,7 @@
"moment": "^2.15.0",
"multer": "^1.2.0",
"mysql": "^2.11.1",
"node-7z": "^0.4.0",
"pygmentize-bundled-cached": "^1.1.0",
"request": "^2.74.0",
"request-promise": "^4.1.1",

Loading…
Cancel
Save