Fix zip blocks the main thread

models/file.js

@@ -56,28 +56,43 @@ class File extends Model {
     return syzoj.utils.resolvePath(syzoj.config.upload_dir, type, md5);
   }
 
-  static async upload(pathOrData, type, noLimit) {
+  // data: Array of { filename: string, data: buffer or string }
+  static async zipFiles(files) {
+    let tmp = require('tmp-promise');
+    let dir = await tmp.dir(), path = require('path'), fs = require('fs-extra');
+    let filenames = await files.mapAsync(async file => {
+      let fullPath = path.join(dir.path, file.filename);
+      await fs.writeFileAsync(fullPath, file.data);
+      return fullPath;
+    });
+
+    let p7zip = new (require('node-7z')), zipFile = await tmp.tmpName() + '.zip';
+    console.log(await p7zip.add(zipFile, filenames));
+
+    await fs.removeAsync(dir.path);
+
+    return zipFile;
+  }
+
+  static async upload(path, type, noLimit) {
     let fs = Promise.promisifyAll(require('fs-extra'));
 
-    let buf = Buffer.isBuffer(pathOrData) ? pathOrData : await fs.readFileAsync(pathOrData);
+    let buf = await fs.readFileAsync(path);
 
     if (!noLimit && buf.length > syzoj.config.limit.data_size) throw new ErrorMessage('数据包太大。');
 
     try {
-      let AdmZip = require('adm-zip');
-      let zip = new AdmZip(buf);
+      let p7zip = new (require('node-7z'));
       this.unzipSize = 0;
-      for (let x of zip.getEntries()) this.unzipSize += x.header.size;
+      await p7zip.list(path).progress(files => {
+        for (let file of files) this.unzipSize += file.size;
+      });
     } catch (e) {
       this.unzipSize = null;
     }
 
     let key = syzoj.utils.md5(buf);
-    if (Buffer.isBuffer(pathOrData)) {
-      await fs.writeFileAsync(File.resolvePath(type, key), pathOrData);
-    } else {
-      await fs.moveAsync(pathOrData, File.resolvePath(type, key), { overwrite: true });
-    }
+    await fs.moveAsync(path, File.resolvePath(type, key), { overwrite: true });
 
     let file = await File.findOne({ where: { md5: key } });
     if (!file) {
@@ -94,14 +109,11 @@ class File extends Model {
   async getUnzipSize() {
     if (this.unzipSize === undefined)  {
       try {
-        let fs = Promise.promisifyAll(require('fs-extra'));
-        let buf = await fs.readFileAsync(this.getPath());
-
-        let AdmZip = require('adm-zip');
-        let zip = new AdmZip(buf);
-
+        let p7zip = new (require('node-7z'));
         this.unzipSize = 0;
-        for (let x of zip.getEntries()) this.unzipSize += x.header.size;
+        await p7zip.list(this.getPath()).progress(files => {
+          for (let file of files) this.unzipSize += file.size;
+        });
       } catch (e) {
         this.unzipSize = null;
       }

models/problem.js

@@ -319,56 +319,63 @@ class Problem extends Model {
   }
 
   async updateTestdata(path, noLimit) {
-    let AdmZip = require('adm-zip');
-    let zip = new AdmZip(path);
+    await syzoj.utils.lock(['Problem::Testdata', this.id], async () => {
+      let p7zip = new (require('node-7z'));
 
-    let unzipSize = 0;
-    for (let x of zip.getEntries()) unzipSize += x.header.size;
-    if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
+      let unzipSize = 0;
+      await p7zip.list(path).progress(files => {
+        for (let file of files) unzipSize += file.size;
+      });
+      if (!noLimit && unzipSize > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
 
-    let dir = this.getTestdataPath();
-    let fs = Promise.promisifyAll(require('fs-extra'));
-    await fs.removeAsync(dir);
-    await fs.ensureDirAsync(dir);
+      let dir = this.getTestdataPath();
+      let fs = Promise.promisifyAll(require('fs-extra'));
+      await fs.removeAsync(dir);
+      await fs.ensureDirAsync(dir);
 
-    zip.extractAllTo(dir);
-    await fs.moveAsync(path, dir + '.zip', { overwrite: true });
+      await p7zip.extract(path, dir);
+      await fs.moveAsync(path, dir + '.zip', { overwrite: true });
+    });
   }
 
   async uploadTestdataSingleFile(filename, filepath, size, noLimit) {
-    let dir = this.getTestdataPath();
-    let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
-    await fs.ensureDirAsync(dir);
-
-    let oldSize = 0;
-    let list = await this.listTestdata();
-    if (list) {
-      for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
-    }
+    await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
+      let dir = this.getTestdataPath();
+      let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
+      await fs.ensureDirAsync(dir);
 
-    if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
+      let oldSize = 0;
+      let list = await this.listTestdata();
+      if (list) {
+        for (let file of list.files) if (file.filename !== filename) oldSize += file.size;
+      }
+
+      if (!noLimit && oldSize + size > syzoj.config.limit.testdata) throw new ErrorMessage('数据包太大。');
 
-    await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true });
-    await fs.removeAsync(dir + '.zip');
+      await fs.moveAsync(filepath, path.join(dir, filename), { overwrite: true });
+      await fs.removeAsync(dir + '.zip');
+    });
   }
 
   async deleteTestdataSingleFile(filename) {
-    let dir = this.getTestdataPath();
-    let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
-    await fs.removeAsync(path.join(dir, filename));
-    await fs.removeAsync(dir + '.zip');
+    await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
+      let dir = this.getTestdataPath();
+      let fs = Promise.promisifyAll(require('fs-extra')), path = require('path');
+      await fs.removeAsync(path.join(dir, filename));
+      await fs.removeAsync(dir + '.zip');
+    });
   }
 
   async makeTestdataZip() {
-    let dir = this.getTestdataPath();
-    if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。');
+    await syzoj.utils.lock(['Promise::Testdata', this.id], async () => {
+      let dir = this.getTestdataPath();
+      if (!await syzoj.utils.isDir(dir)) throw new ErrorMessage('无测试数据。');
 
-    let AdmZip = require('adm-zip');
-    let zip = new AdmZip();
+      let p7zip = new (require('node-7z'));
 
-    let list = await this.listTestdata();
-    for (let file of list.files) zip.addLocalFile(require('path').join(dir, file.filename), '', file.filename);
-    zip.writeZip(dir + '.zip');
+      let list = await this.listTestdata(), path = require('path');
+      await p7zip.add(dir + '.zip', list.files.map(file => path.join(dir, file.filename)));
+    });
   }
 
   async hasSpecialJudge() {

modules/problem.js

@@ -568,27 +568,21 @@ app.post('/problem/:id/submit', app.multer.fields([{ name: 'answer', maxCount: 1
 
     let judge_state;
     if (problem.type === 'submit-answer') {
-      let pathOrData;
+      let File = syzoj.model('file'), path;
       if (!req.files['answer']) {
         // Submited by editor
         try {
-          let files = JSON.parse(req.body.answer_by_editor);
-          let AdmZip = require('adm-zip');
-          let zip = new AdmZip();
-          for (let file of files) {
-            zip.addFile(file.filename, file.data);
-          }
-          pathOrData = zip.toBuffer();
+          path = await File.zipFiles(JSON.parse(req.body.answer_by_editor));
         } catch (e) {
+          console.log(e);
           throw new ErrorMessage('无法解析提交数据。');
         }
       } else {
         if (req.files['answer'][0].size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');
-        pathOrData = req.files['answer'][0].path;
+        path = req.files['answer'][0].path;
       }
 
-      let File = syzoj.model('file');
-      let file = await File.upload(pathOrData, 'answer');
+      let file = await File.upload(path, 'answer');
       let size = await file.getUnzipSize();
 
       if (size > syzoj.config.limit.submit_answer) throw new ErrorMessage('答案文件太大。');

package.json

@@ -23,7 +23,6 @@
   },
   "homepage": "https://github.com/syzoj/syzoj#readme",
   "dependencies": {
-    "adm-zip": "^0.4.7",
     "ansi-to-html": "^0.4.2",
     "async-lock": "^0.3.9",
     "body-parser": "^1.15.2",
@@ -41,6 +40,7 @@
     "moment": "^2.15.0",
     "multer": "^1.2.0",
     "mysql": "^2.11.1",
+    "node-7z": "^0.4.0",
     "pygmentize-bundled-cached": "^1.1.0",
     "request": "^2.74.0",
     "request-promise": "^4.1.1",