diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..d6b1940 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +// 将设置放入此文件中以覆盖默认值和用户设置。 +{ +} \ No newline at end of file diff --git a/daemon-config-example.json b/daemon-config-example.json index e101b02..6abd4df 100644 --- a/daemon-config-example.json +++ b/daemon-config-example.json @@ -1,7 +1,7 @@ { "RabbitMQUrl": "amqp://localhost/", "RedisUrl": "redis://127.0.0.1:6379", - "TestData": "/home/t123yh/syzoj/uploads/testdata", + "TestData": "/home/t123yh/sync2/syzoj/uploads/testdata", "Priority": 1, "DataDisplayLimit": 100 -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 20a35e9..11ba783 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,6 +79,15 @@ "integrity": "sha512-bNVEiBrpEdg5oWz/10gxLUSjQGeBx7HgJHgy2DfR1K7unn260FfRwjVYH/NngQucYeMqsg1hOSo0+heQLJAwZA==", "dev": true }, + "@types/jsonwebtoken": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-7.2.3.tgz", + "integrity": "sha512-cVhxZfVCyTZd1P+2a+xXSR9to7hZTulNRLLCQMVfAevUqx2Ee+EgsiD/7pX8qvdXWP3nWgSoTjKRLMrIpdPVjQ==", + "dev": true, + "requires": { + "@types/node": "7.0.39" + } + }, "@types/klaw": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/klaw/-/klaw-1.3.2.tgz", @@ -170,6 +179,15 @@ "@types/mime": "1.3.1" } }, + "@types/socket.io": { + "version": "1.4.29", + "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-1.4.29.tgz", + "integrity": "sha1-hqazqat4z5qQDO74W5totr6oZxI=", + "dev": true, + "requires": { + "@types/node": "7.0.39" + } + }, "@types/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.0.tgz", @@ -197,6 +215,11 @@ "negotiator": "0.6.1" } }, + "after": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", + "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=" + }, "ajv": { "version": "4.11.8", "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", @@ -217,6 +240,16 @@ "readable-stream": "1.1.14" } }, + "ansi-regex": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz", + "integrity": "sha1-DY6UaWej2BQ/k+JOKYUl/BsiNfk=" + }, + "ansi-styles": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.1.0.tgz", + "integrity": "sha1-6uy/Zs1waIJ2Cy9GkVgrj1XXp94=" + }, "argparse": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz", @@ -243,6 +276,11 @@ "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.2.tgz", "integrity": "sha1-X8w3OSB3VyPP1k1lxkvvU7+eum0=" }, + "arraybuffer.slice": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.6.tgz", + "integrity": "sha1-8zshWfBTKj8xB6JywMz70a0peco=" + }, "asn1": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", @@ -273,6 +311,26 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=" }, + "backo2": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", + "integrity": "sha1-MasayLEpNjRj41s+u2n038+6eUc=" + }, + "base64-arraybuffer": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz", + "integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg=" + }, + "base64id": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz", + "integrity": "sha1-R2iMuZu2gE8OBtPnY7HDLlfY5rY=" + }, + "base64url": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/base64url/-/base64url-2.0.0.tgz", + "integrity": "sha1-6sFuA+oUOO/5Qj1puqNiYu0fcLs=" + }, "bcrypt-pbkdf": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", @@ -282,6 +340,14 @@ "tweetnacl": "0.14.5" } }, + "better-assert": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz", + "integrity": "sha1-QIZrnhueC1W0gYlDEeaPr/rrxSI=", + "requires": { + "callsite": "1.0.0" + } + }, "bindings": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz", @@ -295,6 +361,11 @@ "buffer-more-ints": "0.0.2" } }, + "blob": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz", + "integrity": "sha1-vPEwUspURj8w+fx+lbmkdjCpSSE=" + }, "bluebird": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.0.tgz", @@ -325,6 +396,11 @@ "hoek": "2.16.3" } }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" + }, "buffer-more-ints": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz", @@ -335,11 +411,28 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.4.0.tgz", "integrity": "sha1-fZcZb51br39pNeJZhVSe3SpsIzk=" }, + "callsite": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", + "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA=" + }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, + "chalk": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz", + "integrity": "sha1-Zjs6ZItotV0EaQ1JFnqoN4WPIXQ=", + "requires": { + "ansi-styles": "1.1.0", + "escape-string-regexp": "1.0.5", + "has-ansi": "0.1.0", + "strip-ansi": "0.3.0", + "supports-color": "0.2.0" + } + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -368,6 +461,21 @@ "typical": "2.6.1" } }, + "component-bind": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz", + "integrity": "sha1-AMYIq33Nk4l8AAllGx06jh5zu9E=" + }, + "component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" + }, + "component-inherit": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz", + "integrity": "sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM=" + }, "content-disposition": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", @@ -463,6 +571,15 @@ "jsbn": "0.1.1" } }, + "ecdsa-sig-formatter": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.9.tgz", + "integrity": "sha1-S8kmJ07Dtau1AW5+HWCSGsJisqE=", + "requires": { + "base64url": "2.0.0", + "safe-buffer": "5.1.1" + } + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -473,11 +590,61 @@ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz", "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA=" }, + "engine.io": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz", + "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=", + "requires": { + "accepts": "1.3.3", + "base64id": "1.0.0", + "cookie": "0.3.1", + "debug": "2.6.7", + "engine.io-parser": "2.1.1", + "uws": "0.14.5", + "ws": "2.3.1" + } + }, + "engine.io-client": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz", + "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=", + "requires": { + "component-emitter": "1.2.1", + "component-inherit": "0.0.3", + "debug": "2.6.7", + "engine.io-parser": "2.1.1", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "parsejson": "0.0.3", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "ws": "2.3.1", + "xmlhttprequest-ssl": "1.5.3", + "yeast": "0.1.2" + } + }, + "engine.io-parser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz", + "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=", + "requires": { + "after": "0.8.2", + "arraybuffer.slice": "0.0.6", + "base64-arraybuffer": "0.1.5", + "blob": "0.0.4", + "has-binary2": "1.0.2" + } + }, "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, "esprima": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz", @@ -651,6 +818,34 @@ "har-schema": "1.0.5" } }, + "has-ansi": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-0.1.0.tgz", + "integrity": "sha1-hPJlqujA5qiKEtcCKJS3VoiUxi4=", + "requires": { + "ansi-regex": "0.2.1" + } + }, + "has-binary2": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.2.tgz", + "integrity": "sha1-6D26SfC5vk0CbSc2U1DZ8D9Uvpg=", + "requires": { + "isarray": "2.0.1" + }, + "dependencies": { + "isarray": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz", + "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=" + } + } + }, + "has-cors": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz", + "integrity": "sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk=" + }, "hawk": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", @@ -705,6 +900,11 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" }, + "indexof": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz", + "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10=" + }, "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", @@ -730,11 +930,27 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isemail": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/isemail/-/isemail-1.2.0.tgz", + "integrity": "sha1-vgPfjMPineTSxd9lASY/H6RZXpo=" + }, "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "joi": { + "version": "6.10.1", + "resolved": "https://registry.npmjs.org/joi/-/joi-6.10.1.tgz", + "integrity": "sha1-TVDDGAeRIgAP5fFq8f+OGRe3fgY=", + "requires": { + "hoek": "2.16.3", + "isemail": "1.2.0", + "moment": "2.18.1", + "topo": "1.1.0" + } + }, "js-yaml": { "version": "3.9.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.9.1.tgz", @@ -768,6 +984,14 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, + "jsondiffpatch": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.2.4.tgz", + "integrity": "sha1-1LbFOz/H2htLkcHCrsi5MrdRHVw=", + "requires": { + "chalk": "0.5.1" + } + }, "jsonfile": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-3.0.1.tgz", @@ -781,6 +1005,18 @@ "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" }, + "jsonwebtoken": { + "version": "7.4.2", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-7.4.2.tgz", + "integrity": "sha1-VxuQPAfodcD8WSA9GseGZ9gOCc0=", + "requires": { + "joi": "6.10.1", + "jws": "3.1.4", + "lodash.once": "4.1.1", + "ms": "2.0.0", + "xtend": "4.0.1" + } + }, "jsprim": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", @@ -799,6 +1035,27 @@ } } }, + "jwa": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.1.5.tgz", + "integrity": "sha1-oFUs4CIHQs1S4VN3SjKQXDDnVuU=", + "requires": { + "base64url": "2.0.0", + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.9", + "safe-buffer": "5.1.1" + } + }, + "jws": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.4.tgz", + "integrity": "sha1-+ei5M46KhHJ31kRLFGT2GIDgUKI=", + "requires": { + "base64url": "2.0.0", + "jwa": "1.1.5", + "safe-buffer": "5.1.1" + } + }, "klaw": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/klaw/-/klaw-2.0.0.tgz", @@ -817,6 +1074,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" }, + "lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=" + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -886,6 +1148,11 @@ } } }, + "moment": { + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.18.1.tgz", + "integrity": "sha1-w2GT3Tzhwu7SrbfIAtu8d6gbHA8=" + }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -917,6 +1184,16 @@ "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, + "object-component": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz", + "integrity": "sha1-8MaapQ78lbhmwYb0AKM3acsvEpE=" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -925,6 +1202,30 @@ "ee-first": "1.1.1" } }, + "parsejson": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/parsejson/-/parsejson-0.0.3.tgz", + "integrity": "sha1-q343WfIJ7OmUN5c/fQ8fZK4OZKs=", + "requires": { + "better-assert": "1.0.2" + } + }, + "parseqs": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", + "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=", + "requires": { + "better-assert": "1.0.2" + } + }, + "parseuri": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz", + "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=", + "requires": { + "better-assert": "1.0.2" + } + }, "parseurl": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz", @@ -1150,6 +1451,62 @@ "hoek": "2.16.3" } }, + "socket.io": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.0.3.tgz", + "integrity": "sha1-Q1nwaiSTOua9CHeYr3jGgOrjReM=", + "requires": { + "debug": "2.6.7", + "engine.io": "3.1.0", + "object-assign": "4.1.1", + "socket.io-adapter": "1.1.1", + "socket.io-client": "2.0.3", + "socket.io-parser": "3.1.2" + } + }, + "socket.io-adapter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz", + "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=" + }, + "socket.io-client": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.0.3.tgz", + "integrity": "sha1-bK9K/5+FsZ/ZG2zhPWmttWT4hzs=", + "requires": { + "backo2": "1.0.2", + "base64-arraybuffer": "0.1.5", + "component-bind": "1.0.0", + "component-emitter": "1.2.1", + "debug": "2.6.7", + "engine.io-client": "3.1.1", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "object-component": "0.0.3", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "socket.io-parser": "3.1.2", + "to-array": "0.1.4" + } + }, + "socket.io-parser": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.1.2.tgz", + "integrity": "sha1-28IoIVH8T6675Aru3Ady66YZ9/I=", + "requires": { + "component-emitter": "1.2.1", + "debug": "2.6.7", + "has-binary2": "1.0.2", + "isarray": "2.0.1" + }, + "dependencies": { + "isarray": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz", + "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=" + } + } + }, "source-map": { "version": "0.5.6", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", @@ -1215,6 +1572,19 @@ "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" }, + "strip-ansi": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz", + "integrity": "sha1-JfSOoiynkYfzF0pNuHWTR7sSYiA=", + "requires": { + "ansi-regex": "0.2.1" + } + }, + "supports-color": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz", + "integrity": "sha1-2S3iaU6z9nMjlz1649i1W0wiGQo=" + }, "tar": { "version": "3.1.9", "resolved": "https://registry.npmjs.org/tar/-/tar-3.1.9.tgz", @@ -1235,6 +1605,19 @@ "typical": "2.6.1" } }, + "to-array": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz", + "integrity": "sha1-F+bBH3PdTz10zaek/zI46a2b+JA=" + }, + "topo": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/topo/-/topo-1.1.0.tgz", + "integrity": "sha1-6ddRYV0buH3IZdsYL6HKCl71NtU=", + "requires": { + "hoek": "2.16.3" + } + }, "tough-cookie": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz", @@ -1277,6 +1660,11 @@ "resolved": "https://registry.npmjs.org/typical/-/typical-2.6.1.tgz", "integrity": "sha1-XAgOXWYcu+OCWdLnCjxyU+hziB0=" }, + "ultron": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz", + "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ=" + }, "universalify": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz", @@ -1297,6 +1685,12 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, + "uws": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/uws/-/uws-0.14.5.tgz", + "integrity": "sha1-Z6rzPEaypYel9mZtAPdpEyjxSdw=", + "optional": true + }, "vary": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.1.tgz", @@ -1332,10 +1726,41 @@ "stack-trace": "0.0.10" } }, + "ws": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz", + "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=", + "requires": { + "safe-buffer": "5.0.1", + "ultron": "1.1.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz", + "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c=" + } + } + }, + "xmlhttprequest-ssl": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.3.tgz", + "integrity": "sha1-GFqIjATspGw+QHDZn3tJ3jUomS0=" + }, + "xtend": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, "yallist": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz", "integrity": "sha1-hFK0u36Dx8GI2AQcGoN8dz1ti7k=" + }, + "yeast": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz", + "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk=" } } } diff --git a/package.json b/package.json index f527e13..b6e7594 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "fs-extra": "^3.0.1", "get-folder-size": "^1.0.0", "js-yaml": "^3.9.1", + "jsondiffpatch": "^0.2.4", + "jsonwebtoken": "^7.4.2", "klaw": "^2.0.0", "lockfile": "^1.0.3", "lodash": "^4.17.4", @@ -26,6 +28,7 @@ "request": "^2.81.0", "request-promise": "^4.2.1", "simple-sandbox": "^0.3.3", + "socket.io": "^2.0.3", "source-map-support": "^0.4.15", "tar": "^3.1.9", "uuid": "^3.1.0", @@ -39,6 +42,7 @@ "@types/express": "^4.0.36", "@types/fs-extra": "^3.0.3", "@types/js-yaml": "^3.9.0", + "@types/jsonwebtoken": "^7.2.3", "@types/klaw": "^1.3.2", "@types/lodash": "^4.14.71", "@types/msgpack-lite": "^0.1.4", @@ -47,6 +51,7 @@ "@types/redlock": "0.0.31", "@types/request": "^2.0.0", "@types/request-promise": "^4.1.36", + "@types/socket.io": "^1.4.29", "@types/uuid": "^3.4.0", "@types/winston": "^2.3.4", "typescript": "^2.3.4" diff --git a/runner-shared-config-example.json b/runner-shared-config-example.json index 6a0279a..4a93bd2 100644 --- a/runner-shared-config-example.json +++ b/runner-shared-config-example.json @@ -1,7 +1,7 @@ { "RabbitMQUrl": "amqp://localhost/", "RedisUrl": "redis://127.0.0.1:6379", - "TestData": "/home/t123yh/syzoj/uploads/testdata", + "TestData": "/home/t123yh/sync2/syzoj/uploads/testdata", "Priority": 1, "DebugMessageDisplayLimit": 5000, "OutputLimit": 104857600, @@ -17,4 +17,4 @@ "SandboxUser": "nobody", "SandboxRoot": "/home/t123yh/alpine", "BinaryDirectory": "/home/t123yh/syzoj-bin" -} \ No newline at end of file +} diff --git a/src/daemon-frontend-syzoj/config.ts b/src/daemon-frontend-syzoj/config.ts index cb5f34e..b9038ee 100644 --- a/src/daemon-frontend-syzoj/config.ts +++ b/src/daemon-frontend-syzoj/config.ts @@ -1,6 +1,7 @@ -import * as commandLineArgs from 'command-line-args'; -import * as fs from 'fs'; -import * as winston from 'winston'; +import commandLineArgs = require('command-line-args'); +import fs = require('fs'); +import winston = require('winston'); +import { configureWinston } from '../winston-common'; export interface ConfigStructure { rabbitMQ: string; @@ -28,8 +29,4 @@ export const globalConfig: ConfigStructure = { token: configJSON.Token } -if (options.verbose) { - (winston as any).level = 'debug'; -} else { - (winston as any).level = 'warn'; -} \ No newline at end of file +configureWinston(options.verbose); \ No newline at end of file diff --git a/src/daemon-frontend-syzoj/taskRouter.ts b/src/daemon-frontend-syzoj/daemonRouter.ts similarity index 84% rename from src/daemon-frontend-syzoj/taskRouter.ts rename to src/daemon-frontend-syzoj/daemonRouter.ts index 2ef472a..9b04f71 100644 --- a/src/daemon-frontend-syzoj/taskRouter.ts +++ b/src/daemon-frontend-syzoj/daemonRouter.ts @@ -1,4 +1,5 @@ import express = require('express'); +import winston = require('winston'); import { globalConfig as Cfg } from './config'; import { pushTask } from './rmq'; @@ -18,6 +19,7 @@ taskRouter.put('/task', (req, res) => { return res.sendStatus(400); } try { + winston.info("Got task: " + req.body); pushTask(req.body); return res.status(200).send('OK'); } catch (err) { diff --git a/src/daemon-frontend-syzoj/index.ts b/src/daemon-frontend-syzoj/index.ts index 303cced..aec0bc3 100644 --- a/src/daemon-frontend-syzoj/index.ts +++ b/src/daemon-frontend-syzoj/index.ts @@ -3,32 +3,65 @@ require('source-map-support').install(); import express = require('express'); import bodyParser = require('body-parser'); import Bluebird = require('bluebird'); -import url = require('url'); +import urlLib = require('url'); import rp = require('request-promise'); import winston = require('winston'); +import http = require('http'); import { globalConfig as Cfg } from './config'; -import { connect, waitForResult } from './rmq'; +import { connect, waitForResult, waitForProgress } from './rmq'; import { convertResult } from '../judgeResult'; -import taskRouter from './taskRouter'; +import { ProgressReportType, OverallResult, TaskStatus, CompilationResult } from '../interfaces'; +import taskRouter from './daemonRouter'; +import { initializeSocketIO, createTask, updateCompileStatus, updateProgress, updateResult } from './socketio'; const app = express(); app.use(bodyParser.json()); app.use('/daemon', taskRouter); + (async () => { await connect(); await waitForResult(async (result) => { - await rp(url.resolve(Cfg.remoteUrl, "api/v2/judge/update2"), { - method: 'POST', - body: convertResult(result.taskId, result.progress), - headers: { - Token: Cfg.token - }, - json: true, - simple: true - }); + winston.info("Reporting...", result); + + const submit = async function (url, obj) { + winston.debug(`POST ${Cfg.remoteUrl}, data = ${JSON.stringify(obj)}`); + await rp(urlLib.resolve(Cfg.remoteUrl, url), { + method: 'POST', + body: obj, + headers: { + Token: Cfg.token + }, + json: true, + simple: true + }); + } + + if (result.type === ProgressReportType.Finished) { + await submit("api/v2/judge/finished", convertResult(result.taskId, result.progress as OverallResult)); + } else if (result.type === ProgressReportType.Compiled) { + await submit("api/v2/judge/compiled", { + taskId: result.taskId, + result: result.progress + }); + } else { + + } + winston.verbose("Reported."); + }); + await waitForProgress(async (result) => { + if (result.type === ProgressReportType.Started) { + createTask(result.taskId); + } else if (result.type === ProgressReportType.Compiled) { + updateCompileStatus(result.taskId, result.progress as CompilationResult); + } else if (result.type === ProgressReportType.Progress) { + updateProgress(result.taskId, result.progress as OverallResult); + } else if (result.type === ProgressReportType.Finished) { + updateResult(result.taskId, result.progress as OverallResult); + } }); })().then(() => { - app.listen(Cfg.listen.port, Cfg.listen.host); + const server = http.createServer(app); + server.listen(Cfg.listen.port, Cfg.listen.host); }); \ No newline at end of file diff --git a/src/daemon-frontend-syzoj/rmq.ts b/src/daemon-frontend-syzoj/rmq.ts index dd89e41..805fccd 100644 --- a/src/daemon-frontend-syzoj/rmq.ts +++ b/src/daemon-frontend-syzoj/rmq.ts @@ -5,6 +5,7 @@ import winston = require('winston'); import util = require('util'); import { cleanUp } from './cleanup'; import * as rmqCommon from '../rmq-common'; +import requestErrors = require('request-promise/errors'); import { JudgeResult, ProgressReportData } from '../interfaces'; let amqpConnection: amqp.Connection; @@ -17,6 +18,7 @@ export async function connect() { publicChannel = await newChannel(); await rmqCommon.assertJudgeQueue(publicChannel); await rmqCommon.assertResultReportQueue(publicChannel); + await rmqCommon.assertProgressReportExchange(publicChannel); amqpConnection.on('error', (err) => { winston.error(`RabbitMQ connection failure: ${err.toString()}`); cleanUp(2); @@ -38,35 +40,25 @@ export function pushTask(task: any) { } export async function waitForResult(handle: (result: ProgressReportData) => Promise) { - const channel = await newChannel(); - channel.prefetch(1); - await channel.consume(rmqCommon.resultReportQueueName, (msg: amqp.Message) => { - winston.info(`Got result from queue`); - (async () => { - const data = msgpack.decode(msg.content); - await handle(data); - })().then(async () => { - channel.ack(msg); - }, async (err) => { - winston.warn(`Failed to process message ${err.toString()}, try again`); - setTimeout(() => { channel.nack(msg, false, true) }, 500); - }); - }); + await rmqCommon.waitForTask(amqpConnection, rmqCommon.resultReportQueueName, null, (err) => { + if (err instanceof requestErrors.RequestError || err instanceof requestErrors.StatusCodeError || err instanceof requestErrors.TransformError) { + return true; + } else return false; + }, handle); } export async function waitForProgress(handle: (result: ProgressReportData) => Promise) { const channel = await newChannel(); - channel.prefetch(1); - await channel.consume(rmqCommon.resultReportQueueName, (msg: amqp.Message) => { - winston.info(`Got result from queue`); - (async () => { - const data = msgpack.decode(msg.content); - await handle(data); - })().then(async () => { - channel.ack(msg); + const queueName = (await channel.assertQueue('', { exclusive: true })).queue; + await channel.bindQueue(queueName, rmqCommon.progressExchangeName, ''); + await channel.consume(queueName, (msg: amqp.Message) => { + const data = msgpack.decode(msg.content) as ProgressReportData; + winston.verbose(`Got result from progress exchange, id: ${data.taskId}`); + + handle(data).then(async () => { + channel.ack(msg) }, async (err) => { - winston.warn(`Failed to process message ${err.toString()}, try again`); - setTimeout(() => { channel.nack(msg, false, true) }, 500); + channel.nack(msg, false, false); }); }); } diff --git a/src/daemon-frontend-syzoj/socketio.ts b/src/daemon-frontend-syzoj/socketio.ts new file mode 100644 index 0000000..040d077 --- /dev/null +++ b/src/daemon-frontend-syzoj/socketio.ts @@ -0,0 +1,190 @@ +import http = require('http'); +import socketio = require('socket.io'); +import diff = require('jsondiffpatch'); +import jwt = require('jsonwebtoken'); + +import { globalConfig as Cfg } from './config'; +import { convertResult } from '../judgeResult'; +import { JudgeResult, TaskStatus, CompilationResult, OverallResult } from '../interfaces'; + +interface JudgeData { + running: boolean; + current?: OverallResult; +} + +interface RoughResult { + result: string; + score: number; + time: number; + memory: number; +} + +let ioInstance: SocketIO.Server; +let detailProgressNamespace: SocketIO.Namespace; +// To do: find a better name +let roughProgressNamespace: SocketIO.Namespace; +// Provide support for NOI contests in which participants +// can only see whether his / her submission is successfully compiled. +let compileProgressNamespace: SocketIO.Namespace; + +const currentJudgeList: JudgeData[] = []; +const finishedJudgeList = {}; + +export function initializeSocketIO(s: http.Server) { + ioInstance = socketio(http); + detailProgressNamespace = ioInstance.of('/detail'); + roughProgressNamespace = ioInstance.of('/rough'); + compileProgressNamespace = ioInstance.of('/compile'); + + // TODO: deduplicate the following code. + detailProgressNamespace.on('connection', (socket) => { + socket.on('join', (reqJwt, cb) => { + let req; + try { + req = jwt.verify(reqJwt, Cfg.token); + } catch (err) { + cb({ + ok: false, + message: err.toString() + }); + return; + } + const taskId = req.taskId; + socket.join(taskId.toString()); + if (finishedJudgeList[taskId]) { + cb({ + ok: true, + finished: true + }); + } else { + cb({ + ok: true, + finished: false, + current: currentJudgeList[taskId] + }); + } + }); + }); + + roughProgressNamespace.on('connection', (socket) => { + socket.on('join', (reqJwt, cb) => { + let req; + try { + req = jwt.verify(reqJwt, Cfg.token); + } catch (err) { + cb({ + ok: false, + message: err.toString() + }); + return; + } + const taskId = req.taskId; + socket.join(taskId.toString()); + if (currentJudgeList[taskId]) { + cb({ + ok: true, + running: true, + finished: false + }); + } else if (finishedJudgeList[taskId]) { + // This is not likely to happen. If some task is processed, + // The client should not process it. + cb({ + ok: true, + running: false, + finished: true + }); + } else { + cb({ + ok: true, + running: false, + finished: false + }) + } + }); + }); + + compileProgressNamespace.on('connection', (socket) => { + socket.on('join', (reqJwt, cb) => { + let req; + try { + req = jwt.verify(reqJwt, Cfg.token); + } catch (err) { + cb({ + ok: false, + message: err.toString() + }); + return; + } + const taskId = req.taskId; + socket.join(taskId.toString()); + if (finishedJudgeList[taskId]) { + cb({ + ok: true, + finished: true + }); + } else if (currentJudgeList[taskId] + && currentJudgeList[taskId].current + && currentJudgeList[taskId].current.compile) { + cb({ + ok: true, + finished: true + }); + } else { + cb({ + ok: true, + finished: false + }); + } + }); + }); +} + + +export function createTask(taskId: number) { + detailProgressNamespace.to(taskId.toString()).emit("start"); + roughProgressNamespace.to(taskId.toString()).emit("start"); + compileProgressNamespace.to(taskId.toString()).emit("start"); + currentJudgeList[taskId] = { running: true, current: null }; +} + +export function updateCompileStatus(taskId: number, result: CompilationResult) { + compileProgressNamespace.to(taskId.toString()).emit('compiled', { + taskId: taskId, + result: { + ok: result.status === TaskStatus.Done, + message: result.message + } + }); +} + +export function updateProgress(taskId: number, data: OverallResult) { + // currentJudgeList[taskId].current = data; + const original = currentJudgeList[taskId].current; + const delta = diff.diff(original, data); + detailProgressNamespace.to(taskId.toString()).emit('update', { + taskId: taskId, + delta: delta + }); + currentJudgeList[taskId].current = data; +} + +export function updateResult(taskId: number, data: OverallResult) { + const finalResult = convertResult(taskId, data); + const roughResult = { + result: finalResult.statusString, + time: finalResult.time, + memory: finalResult.memory, + score: finalResult.score + }; + roughProgressNamespace.to(taskId.toString()).emit('finish', { + taskId: taskId, + result: roughResult + }); + detailProgressNamespace.to(taskId.toString()).emit('finish', { + taskId: taskId, + result: data + }); + delete currentJudgeList[taskId]; + finishedJudgeList[taskId] = true; +} \ No newline at end of file diff --git a/src/daemon/config.ts b/src/daemon/config.ts index 8f6cf87..6648be0 100644 --- a/src/daemon/config.ts +++ b/src/daemon/config.ts @@ -1,6 +1,7 @@ -import * as commandLineArgs from 'command-line-args'; -import * as fs from 'fs'; -import * as winston from 'winston'; +import commandLineArgs = require('command-line-args'); +import fs = require('fs'); +import winston = require('winston'); +import { configureWinston } from '../winston-common'; export interface ConfigStructure { rabbitMQ: string; @@ -30,9 +31,4 @@ export const globalConfig: ConfigStructure = { dataDisplayLimit: configJSON.DataDisplayLimit } -if (options.verbose) { - // winston.transports.Console.level = 'debug'; - (winston as any).level = 'debug'; -} else { - (winston as any).level = 'warn'; -} \ No newline at end of file +configureWinston(options.verbose); \ No newline at end of file diff --git a/src/daemon/index.ts b/src/daemon/index.ts index 589f77f..aafa451 100644 --- a/src/daemon/index.ts +++ b/src/daemon/index.ts @@ -5,18 +5,22 @@ import { globalConfig as Cfg } from './config'; import util = require('util'); import rmq = require('./rmq'); import { judge } from './judge'; -import { JudgeResult, ErrorType, ProgressReportType } from '../interfaces'; +import { JudgeResult, ErrorType, ProgressReportType, OverallResult } from '../interfaces'; (async function () { winston.info("Daemon starts."); await rmq.connect(); winston.info("Start consuming the queue."); await rmq.waitForTask(async (task) => { - let result: JudgeResult; + let result: OverallResult; try { await rmq.reportProgress({ taskId: task.taskId, type: ProgressReportType.Started, progress: null }); result = await judge(task, async (progress) => { await rmq.reportProgress({ taskId: task.taskId, type: ProgressReportType.Progress, progress: progress }); + }, async (progress) => { + const data = { taskId: task.taskId, type: ProgressReportType.Compiled, progress: progress }; + await rmq.reportProgress(data); + await rmq.reportResult(data); }); } catch (err) { winston.warn(`Judge error!!! TaskId: ${task.taskId}`, err); diff --git a/src/daemon/interfaces.ts b/src/daemon/interfaces.ts index 5047b1f..5c31769 100644 --- a/src/daemon/interfaces.ts +++ b/src/daemon/interfaces.ts @@ -1,5 +1,5 @@ import { Language } from '../languages'; -import { FileContent, TaskStatus, TaskResult } from '../interfaces'; +import { FileContent, TaskStatus, TestcaseResult } from '../interfaces'; export enum ProblemType { Standard = 1, @@ -41,7 +41,7 @@ export enum SubtaskScoringType { Multiple } -export interface TestCaseJudge { +export interface TestcaseJudge { input?: string; output?: string; userOutputFile?: string; @@ -51,7 +51,7 @@ export interface TestCaseJudge { export interface SubtaskJudge { type: SubtaskScoringType; score: number; - cases: TestCaseJudge[]; + cases: TestcaseJudge[]; } export interface Executable { diff --git a/src/daemon/judge/compile.ts b/src/daemon/judge/compile.ts index a8bf619..9636967 100644 --- a/src/daemon/judge/compile.ts +++ b/src/daemon/judge/compile.ts @@ -2,17 +2,21 @@ import { Language } from '../../languages'; import * as redis from '../redis'; import * as rmq from '../rmq'; import { codeFingerprint } from '../../utils'; -import { CompileResult, TaskResult, RPCTaskType, RPCRequest, CompileTask, FileContent } from '../../interfaces'; +import { CompilationResult, TestcaseResult, RPCTaskType, TaskStatus, RPCRequest, CompileTask, FileContent } from '../../interfaces'; +import winston = require('winston'); export async function compile( code: string, language: Language, extraFiles: FileContent[] = [], priority: number -): Promise<[string, CompileResult]> { +): Promise<[string, CompilationResult]> { const fingerprint = codeFingerprint(code, language.name); - let result: CompileResult; + winston.debug(`Compiling code, fingerprint = ${fingerprint}`); + let result: CompilationResult; const unlock = await redis.getCompileLock(fingerprint); + winston.debug(`Got redis lock for ${fingerprint}`); try { if (await redis.checkBinaryExistance(fingerprint)) { - result = { status: 0 }; + winston.debug('Binary already exists. Exiting'); + result = { status: TaskStatus.Done }; } else { const task: CompileTask = { code: code, diff --git a/src/daemon/judge/index.ts b/src/daemon/judge/index.ts index 2318cc2..60d7867 100644 --- a/src/daemon/judge/index.ts +++ b/src/daemon/judge/index.ts @@ -1,17 +1,42 @@ import { JudgeTask, ProblemType, TestData, StandardJudgeParameter } from '../interfaces'; -import { judgeStandard } from './standard'; -import {JudgeResult, ErrorType} from '../../interfaces'; +import { StandardJudger } from './standard'; +import { JudgerBase } from './judger-base'; +import { JudgeResult, ErrorType, OverallResult, CompilationResult, TaskStatus, ProgressReportType } from '../../interfaces'; import { readRulesFile } from '../testData'; import { filterPath } from '../../utils'; import winston = require('winston'); - +import rmq = require('../rmq'); +export async function listen() { + await rmq.waitForTask(async (task) => { + let result: OverallResult; + try { + await rmq.reportProgress({ taskId: task.taskId, type: ProgressReportType.Started, progress: null }); + result = await judge(task, async (progress) => { + await rmq.reportProgress({ taskId: task.taskId, type: ProgressReportType.Progress, progress: progress }); + }, async (compileResult) => { + const cresult = { taskId: task.taskId, type: ProgressReportType.Compiled, progress: compileResult }; + await rmq.reportResult(cresult); + await rmq.reportProgress(cresult); + }); + } catch (err) { + winston.warn(`Judge error!!! TaskId: ${task.taskId}`, err); + result = { error: ErrorType.SystemError, systemMessage: `An error occurred.\n${err.toString()}` }; + } + const resultReport = { taskId: task.taskId, type: ProgressReportType.Finished, progress: result }; + await rmq.reportResult(resultReport); + await rmq.reportProgress(resultReport); + }); +} export async function judge( - task: JudgeTask, reportProgress: (p: JudgeResult) => Promise -): Promise { + task: JudgeTask, + reportProgress: (p: OverallResult) => Promise, + reportCompileProgress: (p: CompilationResult) => Promise +): Promise { winston.verbose(`Judging ${task.taskId}`); // Parse test data let testData: TestData = null; try { + winston.debug(`Reading rules file for ${task.testData}...`); testData = await readRulesFile(filterPath(task.testData)); } catch (err) { winston.info(`Error reading test data for ${task.testData}`, err); @@ -22,10 +47,33 @@ export async function judge( return { error: ErrorType.TestDataError, systemMessage: "Testdata unavailable." }; } - // Do things + let judger: JudgerBase; if (task.type === ProblemType.Standard) { - return await judgeStandard(testData, task.param as StandardJudgeParameter, task.priority, reportProgress); + judger = new StandardJudger(testData, task.param as StandardJudgeParameter, task.priority); } else { throw new Error(`Task type not supported`); } + + try { + winston.debug(`Preprocessing testdata for ${task.testData}...`); + await judger.preprocessTestData(); + } catch (err) { + winston.verbose(`Test data ${task.testData} err`, err); + return { error: ErrorType.TestDataError, systemMessage: err.toString() }; + } + + winston.debug(`Compiling...`); + const compileResult = await judger.compile(); + winston.debug(`Reporting compilation progress...`); + await reportCompileProgress(compileResult); + if (compileResult.status !== TaskStatus.Done) { + winston.verbose(`Compilation error: ${compileResult.message}`); + return { + compile: compileResult + }; + } + winston.debug(`Judging...`); + const judgeResult = await judger.judge(r => reportProgress({ compile: compileResult, judge: r })); + + return { compile: compileResult, judge: judgeResult }; } \ No newline at end of file diff --git a/src/daemon/judge/judger-base.ts b/src/daemon/judge/judger-base.ts new file mode 100644 index 0000000..39b039c --- /dev/null +++ b/src/daemon/judge/judger-base.ts @@ -0,0 +1,121 @@ +import { TestData, SubtaskScoringType, TestcaseJudge } from '../interfaces'; +import { CompilationResult, JudgeResult, TaskStatus, SubtaskResult, TestcaseDetails } from '../../interfaces'; +import { Language } from '../../languages'; +import { compile } from './compile'; +import winston = require('winston'); +import _ = require('lodash'); + +const globalFullScore = 100; +function calculateSubtaskScore(scoring: SubtaskScoringType, scores: number[]): number { + if (scoring === SubtaskScoringType.Minimum) { + return _.min(scores); + } else if (scoring === SubtaskScoringType.Multiple) { + return _.reduce(scores, + (res, cur) => res * cur, 1); + } else if (scoring === SubtaskScoringType.Summation) { + return _.sum(scores) / scores.length; + } +} + +export abstract class JudgerBase { + priority: number; + testData: TestData; + + constructor(t: TestData, p: number) { + this.priority = p; + this.testData = t; + } + + abstract preprocessTestData(): Promise; + + abstract compile(): Promise; + + async judge(reportProgressResult: (p: JudgeResult) => Promise): Promise { + const results: SubtaskResult[] = this.testData.subtasks.map(t => ({ + cases: t.cases.map(j => ({ + status: TaskStatus.Waiting + })), + status: TaskStatus.Waiting + })); + const reportProgress = function () { + reportProgressResult({ status: TaskStatus.Running, subtasks: results }); + } + winston.debug(`Totally ${results.length} subtasks.`); + + const judgeTasks: Promise[] = []; + for (let subtaskIndex = 0; subtaskIndex < this.testData.subtasks.length; subtaskIndex++) { + const currentResult = results[subtaskIndex]; + const currentTask = this.testData.subtasks[subtaskIndex]; + + judgeTasks.push((async () => { + // Type minimum is skippable, run one by one + if (currentTask.type !== SubtaskScoringType.Summation) { + let skipped: boolean = true; + for (let index = 0; index < currentTask.cases.length; index++) { + const currentTaskResult = currentResult.cases[index]; + if (skipped) { + currentTaskResult.status = TaskStatus.Skipped; + } else { + winston.verbose(`Judging ${subtaskIndex}, case ${index}.`); + let score = 0; + try { + const taskJudge = await this.judgeTestcase(currentTask.cases[index], async () => { + currentTaskResult.status = TaskStatus.Running; + currentResult.status = TaskStatus.Running; + await reportProgress(); + }); + currentTaskResult.status = TaskStatus.Done; + currentTaskResult.result = taskJudge; + score = taskJudge.scoringRate; + } catch (err) { + currentTaskResult.status = TaskStatus.Failed; + currentTaskResult.errorMessage = err.toString(); + winston.warn(`Task runner error: ${err.toString()} (subtask ${subtaskIndex}, case ${index})`); + } + if (score === 0 || score === NaN) { + winston.debug(`Subtask ${subtaskIndex}, case ${index}: zero, skipping the rest.`); + skipped = true; + } + await reportProgress(); + } + } + } else { + // Non skippable, run all immediately + const caseTasks: Promise[] = []; + for (let index = 0; index < currentTask.cases.length; index++) { + caseTasks.push((async () => { + const currentTaskResult = currentResult.cases[index]; + winston.verbose(`Judging ${subtaskIndex}, case ${index}.`); + try { + currentTaskResult.result = await this.judgeTestcase(currentTask.cases[index], async () => { + currentTaskResult.status = TaskStatus.Running; + currentResult.status = TaskStatus.Running; + await reportProgress(); + }); + currentTaskResult.status = TaskStatus.Done; + } catch (err) { + currentTaskResult.status = TaskStatus.Failed; + currentTaskResult.errorMessage = err.toString(); + winston.warn(`Task runner error: ${err.toString()} (subtask ${subtaskIndex}, case ${index})`); + } + await reportProgress(); + })()); + } + await Promise.all(caseTasks); + } + if (currentResult.cases.some(c => c.status === TaskStatus.Failed)) { + // If any testcase has failed, the score is invaild. + currentResult.score = NaN; + currentResult.status = TaskStatus.Failed; + } else { + currentResult.score = calculateSubtaskScore(currentTask.type, currentResult.cases.map(c => c.result ? c.result.scoringRate : 0)) * currentTask.score; + currentResult.status = TaskStatus.Done; + } + winston.verbose(`Subtask ${subtaskIndex}, finished`); + })()); + } + await Promise.all(judgeTasks); + return { status: TaskStatus.Done, subtasks: results }; + } + protected abstract judgeTestcase(curCase: TestcaseJudge, started: () => Promise): Promise; +} \ No newline at end of file diff --git a/src/daemon/judge/process.ts b/src/daemon/judge/process.ts index 7ff1c80..6f3fde1 100644 --- a/src/daemon/judge/process.ts +++ b/src/daemon/judge/process.ts @@ -1,5 +1,5 @@ -import { TestData, StandardJudgeParameter, SubtaskJudge, TestCaseJudge, SubtaskScoringType } from '../interfaces'; -import { SubtaskResult, TestCaseDetails, TaskStatus, TestCaseResult, JudgeResult } from '../../interfaces'; +import { TestData, StandardJudgeParameter, SubtaskJudge, TestcaseJudge, SubtaskScoringType } from '../interfaces'; +import { SubtaskResult, TestcaseDetails, TaskStatus, TestcaseResult, JudgeResult } from '../../interfaces'; import { globalConfig as Cfg } from '../config'; import winston = require('winston'); import _ = require('lodash'); @@ -19,12 +19,13 @@ function calculateSubtaskScore(scoring: SubtaskScoringType, scores: number[]): n export async function processJudgement( subtasks: SubtaskJudge[], reportProgress: (r: SubtaskResult[]) => Promise, - judgeTestCase: (curCase: TestCaseJudge, started: () => Promise) => Promise, + judgeTestcase: (curCase: TestcaseJudge, started: () => Promise) => Promise, ): Promise { const results: SubtaskResult[] = subtasks.map(t => ({ cases: t.cases.map(j => ({ status: TaskStatus.Waiting - })) + })), + status: TaskStatus.Waiting })); winston.debug(`Totally ${results.length} subtasks.`); @@ -45,8 +46,9 @@ export async function processJudgement( winston.verbose(`Judging ${subtaskIndex}, case ${index}.`); let score = 0; try { - const taskJudge = await judgeTestCase(currentTask.cases[index], async () => { + const taskJudge = await judgeTestcase(currentTask.cases[index], async () => { currentTaskResult.status = TaskStatus.Running; + currentResult.status = TaskStatus.Running; await reportProgress(results); }); currentTaskResult.status = TaskStatus.Done; @@ -72,8 +74,9 @@ export async function processJudgement( const currentTaskResult = currentResult.cases[index]; winston.verbose(`Judging ${subtaskIndex}, case ${index}.`); try { - currentTaskResult.result = await judgeTestCase(currentTask.cases[index], async () => { + currentTaskResult.result = await judgeTestcase(currentTask.cases[index], async () => { currentTaskResult.status = TaskStatus.Running; + currentResult.status = TaskStatus.Running; await reportProgress(results); }); currentTaskResult.status = TaskStatus.Done; @@ -90,8 +93,10 @@ export async function processJudgement( if (currentResult.cases.some(c => c.status === TaskStatus.Failed)) { // If any testcase has failed, the score is invaild. currentResult.score = NaN; + currentResult.status = TaskStatus.Failed; } else { currentResult.score = calculateSubtaskScore(currentTask.type, currentResult.cases.map(c => c.result ? c.result.scoringRate : 0)) * currentTask.score; + currentResult.status = TaskStatus.Done; } winston.verbose(`Subtask ${subtaskIndex}, finished`); })()); diff --git a/src/daemon/judge/standard.ts b/src/daemon/judge/standard.ts index c846405..1e389f8 100644 --- a/src/daemon/judge/standard.ts +++ b/src/daemon/judge/standard.ts @@ -1,92 +1,92 @@ -import { TestData, StandardJudgeParameter, TestCaseJudge } from '../interfaces'; -import { TaskStatus, ErrorType, TestCaseDetails, JudgeResult, TaskResult, StandardRunTask, StandardRunResult, RPCTaskType } from '../../interfaces'; +import { TestData, StandardJudgeParameter, TestcaseJudge } from '../interfaces'; +import { TaskStatus, ErrorType, TestcaseDetails, CompilationResult, JudgeResult, TestcaseResult, StandardRunTask, StandardRunResult, RPCTaskType } from '../../interfaces'; import { globalConfig as Cfg } from '../config'; import { cloneObject, readFileLength } from '../../utils'; import { compile } from './compile'; import { Language, getLanguage } from '../../languages'; import { processJudgement } from './process' import { runTask } from '../rmq'; +import { JudgerBase } from './judger-base'; + import pathLib = require('path'); import winston = require('winston'); -export async function judgeStandard( - testData: TestData, - param: StandardJudgeParameter, - priority: number, - reportProgress: (progress: JudgeResult) => Promise -): Promise { - winston.debug("Running standard judging procedure."); +export class StandardJudger extends JudgerBase { + parameters: StandardJudgeParameter; + userCodeLanguage: Language; + spjExecutableName: string = null; + userCodeExecuableName: string = null; - let spjName: string = null; - if (testData.spj != null) { - winston.verbose("Compiling special judge."); - const [spjExecutableName, spjResult] = await compile(testData.spj.sourceCode, testData.spj.language, null, priority); - spjName = spjExecutableName; + constructor(testData: TestData, + param: StandardJudgeParameter, + priority: number) { + super(testData, priority); + this.parameters = param; + this.userCodeLanguage = getLanguage(param.language); + } - if (spjResult.status !== 0) { - winston.verbose("Special judge CE."); - let message = null; - if (spjResult.message != null && spjResult.message !== "") { - message = "===== Special Judge Compilation Message =====" + spjResult.message; + async preprocessTestData(): Promise { + if (this.testData.spj != null) { + winston.verbose("Compiling special judge."); + const [spjExecutableName, spjResult] = await compile(this.testData.spj.sourceCode, + this.testData.spj.language, null, this.priority); + if (spjResult.status !== TaskStatus.Done) { + winston.verbose("Special judge CE: " + spjResult.message); + let message = null; + if (spjResult.message != null && spjResult.message !== "") { + message = "===== Special Judge Compilation Message =====" + spjResult.message; + } + throw new Error(message); + } else { + this.spjExecutableName = spjExecutableName; } - return { error: ErrorType.TestDataError, systemMessage: message }; + } else { + this.spjExecutableName = null; } } - const language = getLanguage(param.language); - winston.verbose("Compiling user program."); - const [executableName, compilationResult] = await compile( - param.code, - language, - testData.extraSourceFiles[language.name], - priority - ); - - if (compilationResult.status !== 0) { - winston.verbose("User program CE."); - let message = null; - if (compilationResult.message != null && compilationResult.message !== "") { - message = compilationResult.message; - } - return { compileStatus: TaskStatus.Failed, compilerMessage: message }; + async compile(): Promise { + const language = getLanguage(this.parameters.language); + const [executableName, compilationResult] = await compile( + this.parameters.code, + language, + this.testData.extraSourceFiles[language.name], + this.priority + ); + this.userCodeExecuableName = executableName; + return compilationResult; } - winston.debug("Start judgement."); - return { - subtasks: await processJudgement( - testData.subtasks, - async (result) => { reportProgress({ subtasks: result }); }, - async (curCase, st) => { - const task: StandardRunTask = { - testDataName: testData.name, - inputData: curCase.input, - answerData: curCase.output, - time: param.timeLimit, - memory: param.memoryLimit, - fileIOInput: param.fileIOInput, - fileIOOutput: param.fileIOOutput, - userExecutableName: executableName, - spjExecutableName: testData.spj ? spjName : null, - }; + async judgeTestcase(curCase: TestcaseJudge, started: () => Promise): Promise { + const task: StandardRunTask = { + testDataName: this.testData.name, + inputData: curCase.input, + answerData: curCase.output, + time: this.parameters.timeLimit, + memory: this.parameters.memoryLimit, + fileIOInput: this.parameters.fileIOInput, + fileIOOutput: this.parameters.fileIOOutput, + userExecutableName: this.userCodeExecuableName, + spjExecutableName: this.spjExecutableName + }; - const [inputContent, outputContent, runResult]: [string, string, StandardRunResult] = await Promise.all([ - readFileLength(pathLib.join(Cfg.testDataDirectory, testData.name, curCase.input), Cfg.dataDisplayLimit), - readFileLength(pathLib.join(Cfg.testDataDirectory, testData.name, curCase.output), Cfg.dataDisplayLimit), - runTask({ type: RPCTaskType.RunStandard, task: task }, priority, st) - ]) as any; + const [inputContent, outputContent, runResult]: [string, string, StandardRunResult] = await Promise.all([ + readFileLength(pathLib.join(Cfg.testDataDirectory, this.testData.name, curCase.input), Cfg.dataDisplayLimit), + readFileLength(pathLib.join(Cfg.testDataDirectory, this.testData.name, curCase.output), Cfg.dataDisplayLimit), + runTask({ type: RPCTaskType.RunStandard, task: task }, this.priority, started) + ]) as any; - return { - type: runResult.result, - time: runResult.time, - memory: runResult.memory, - userError: runResult.userError, - userOutput: runResult.userOutput, - scoringRate: runResult.scoringRate, - spjMessage: runResult.spjMessage, - input: { name: curCase.input, content: inputContent }, - output: { name: curCase.output, content: outputContent }, - systemMessage: runResult.systemMessage - }; - }) - }; + return { + type: runResult.result, + time: runResult.time, + memory: runResult.memory, + userError: runResult.userError, + userOutput: runResult.userOutput, + scoringRate: runResult.scoringRate, + spjMessage: runResult.spjMessage, + input: { name: curCase.input, content: inputContent }, + output: { name: curCase.output, content: outputContent }, + systemMessage: runResult.systemMessage + }; + } } \ No newline at end of file diff --git a/src/daemon/rmq.ts b/src/daemon/rmq.ts index 9576d42..dd09427 100644 --- a/src/daemon/rmq.ts +++ b/src/daemon/rmq.ts @@ -36,33 +36,17 @@ async function newChannel(): Promise { } export async function waitForTask(handle: (task: JudgeTask) => Promise) { - const channel = await newChannel(); - channel.prefetch(1); - await channel.consume(rmqCommon.judgeQueueName, (msg: amqp.Message) => { - const messageId = msg.properties.messageId; - winston.info(`Got judge task`); - (async () => { - const data = msgpack.decode(msg.content) as JudgeTask; - winston.debug(`Data: ${util.inspect(data)}`); - await handle(data); - })().then(async () => { - channel.ack(msg); - }, async (err) => { - // Do not requeue it. - winston.warn(`Failed to process message ${messageId}: ${err.toString()}`); - channel.nack(msg, false, false); - }); - }, { - priority: Cfg.priority - }); + await rmqCommon.waitForTask(amqpConnection, rmqCommon.judgeQueueName, Cfg.priority, () => false, handle); } export async function reportProgress(data: ProgressReportData) { + winston.verbose('Reporting progress', data); const payload = msgpack.encode(data); publicChannel.publish(rmqCommon.progressExchangeName, '', payload); } export async function reportResult(data: ProgressReportData) { + winston.verbose('Reporting result', data); const payload = msgpack.encode(data); publicChannel.sendToQueue(rmqCommon.resultReportQueueName, payload); } diff --git a/src/daemon/testData.ts b/src/daemon/testData.ts index 25f6083..5921851 100644 --- a/src/daemon/testData.ts +++ b/src/daemon/testData.ts @@ -3,7 +3,7 @@ import fse = require('fs-extra'); import pathLib = require('path'); import { Language, languages, getLanguage } from '../languages'; import { compareStringByNumber, tryReadFile, filterPath } from '../utils'; -import { SubtaskScoringType, SubtaskJudge, TestCaseJudge, Executable, TestData } from './interfaces'; +import { SubtaskScoringType, SubtaskJudge, TestcaseJudge, Executable, TestData } from './interfaces'; import { FileContent } from '../interfaces'; import { globalConfig as Cfg } from './config'; diff --git a/src/interfaces.ts b/src/interfaces.ts index 800fb03..52e279c 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -17,8 +17,8 @@ export interface CompileTask { binaryName: string; } -export interface TestCaseDetails { - type: TaskResult; +export interface TestcaseDetails { + type: TestcaseResultType; time: number; memory: number; input: FileContent; @@ -30,15 +30,16 @@ export interface TestCaseDetails { systemMessage: string; }; -export interface TestCaseResult { +export interface TestcaseResult { status: TaskStatus; - result?: TestCaseDetails; + result?: TestcaseDetails; errorMessage?: string; } export interface SubtaskResult { + status: TaskStatus; score?: number; - cases: TestCaseResult[]; + cases: TestcaseResult[]; } export enum ErrorType { @@ -46,12 +47,21 @@ export enum ErrorType { TestDataError } +export interface CompilationResult { + status: TaskStatus; + message?: string; +} + export interface JudgeResult { - error?: ErrorType; - compileStatus?: TaskStatus; + status: TaskStatus; subtasks?: SubtaskResult[]; - compilerMessage?: string; +} + +export interface OverallResult { + error?: ErrorType; systemMessage?: string; + compile?: CompilationResult; + judge?: JudgeResult; } export interface StandardRunResult { @@ -62,7 +72,7 @@ export interface StandardRunResult { scoringRate: number; spjMessage: string; systemMessage: string; - result: TaskResult; + result: TestcaseResultType; } export interface StandardRunTask { @@ -85,7 +95,7 @@ export enum TaskStatus { Skipped = 4 } -export enum TaskResult { +export enum TestcaseResultType { Accepted = 1, WrongAnswer, PartiallyCorrect, @@ -98,12 +108,6 @@ export enum TaskResult { InvalidInteraction } -export interface CompileResult { - // -1: Run failed, 0: OK, others: Compilation Error - status: number; - message?: string; -} - export interface FileContent { content: string, name: string @@ -117,14 +121,15 @@ export enum RPCReplyType { export enum ProgressReportType { Started = 1, - Progress = 2, - Finished = 3 + Compiled = 2, + Progress = 3, + Finished = 4, } export interface ProgressReportData { taskId: number; type: ProgressReportType; - progress: JudgeResult; + progress: OverallResult | CompilationResult; } export interface RPCReply { diff --git a/src/judgeResult.ts b/src/judgeResult.ts index 00cc676..00ba455 100644 --- a/src/judgeResult.ts +++ b/src/judgeResult.ts @@ -1,5 +1,6 @@ import _ = require('lodash'); import winston = require('winston'); +import { JudgeResult, OverallResult, TestcaseResultType, TaskStatus, ErrorType, SubtaskResult, TestcaseResult, TestcaseDetails } from './interfaces'; export interface JudgeResultSubmit { taskId: number; @@ -8,36 +9,34 @@ export interface JudgeResultSubmit { score: number; statusNumber: number; statusString: string; - result: JudgeResult; + result: OverallResult; } -import { JudgeResult, TaskResult, TaskStatus, ErrorType, SubtaskResult, TestCaseResult, TestCaseDetails } from './interfaces'; - const compileError = "Compile Error", systemError = "System Error", testdataError = "No Testdata"; export const statusToString = {}; -statusToString[TaskResult.Accepted] = "Accepted"; -statusToString[TaskResult.WrongAnswer] = "Wrong Answer"; -statusToString[TaskResult.PartiallyCorrect] = "Partially Correct"; -statusToString[TaskResult.MemoryLimitExceeded] = "Memory Limit Exceeded"; -statusToString[TaskResult.TimeLimitExceeded] = "Time Limit Exceeded"; -statusToString[TaskResult.OutputLimitExceeded] = "Output Limit Exceeded"; -statusToString[TaskResult.RuntimeError] = "Runtime Error"; -statusToString[TaskResult.FileError] = "File Error"; -statusToString[TaskResult.JudgementFailed] = "Judgement Failed"; -statusToString[TaskResult.InvalidInteraction] = "Invalid Interaction"; +statusToString[TestcaseResultType.Accepted] = "Accepted"; +statusToString[TestcaseResultType.WrongAnswer] = "Wrong Answer"; +statusToString[TestcaseResultType.PartiallyCorrect] = "Partially Correct"; +statusToString[TestcaseResultType.MemoryLimitExceeded] = "Memory Limit Exceeded"; +statusToString[TestcaseResultType.TimeLimitExceeded] = "Time Limit Exceeded"; +statusToString[TestcaseResultType.OutputLimitExceeded] = "Output Limit Exceeded"; +statusToString[TestcaseResultType.RuntimeError] = "Runtime Error"; +statusToString[TestcaseResultType.FileError] = "File Error"; +statusToString[TestcaseResultType.JudgementFailed] = "Judgement Failed"; +statusToString[TestcaseResultType.InvalidInteraction] = "Invalid Interaction"; -export function firstNonAC(t: TaskResult[]): TaskResult { - if (t.every(v => v === TaskResult.Accepted)) { - return TaskResult.Accepted +export function firstNonAC(t: TestcaseResultType[]): TestcaseResultType { + if (t.every(v => v === TestcaseResultType.Accepted)) { + return TestcaseResultType.Accepted } else { - return t.find(r => r !== TaskResult.Accepted); + return t.find(r => r !== TestcaseResultType.Accepted); } } -export function convertResult(id: number, source: JudgeResult): JudgeResultSubmit { +export function convertResult(id: number, source: OverallResult): JudgeResultSubmit { winston.debug(`Converting result for ${id}`, source); let time = -1, memory = -1, @@ -45,7 +44,7 @@ export function convertResult(id: number, source: JudgeResult): JudgeResultSubmi done = true, statusString = null; - if (source.compileStatus === TaskStatus.Failed) { + if (source.compile && source.compile.status === TaskStatus.Failed) { statusString = compileError; score = 0; } else if (source.error != null) { @@ -56,21 +55,24 @@ export function convertResult(id: number, source: JudgeResult): JudgeResultSubmi } else { statusString = systemError; } - } else if (source.subtasks != null) { - if (source.subtasks.some(s => s.score === NaN)) { + } else if (source.judge != null && source.judge.subtasks != null) { + if (source.judge.subtasks.some(s => s.status === TaskStatus.Failed)) { + winston.debug(`Some subtasks failed, returning system error`); score = NaN; statusString = systemError; } else { - score = _.sum(source.subtasks.map(s => s.score)); + score = _.sum(source.judge.subtasks.map(s => s.score)); - const forEveryTestcase = function (map: (v: TestCaseDetails) => TParam, reduce: (v: TParam[]) => TParam): TParam { - return reduce(source.subtasks.map(s => reduce(s.cases.filter(c => c.result != null).map(c => map(c.result))))); + const forEveryTestcase = function (map: (v: TestcaseDetails) => TParam, reduce: (v: TParam[]) => TParam): TParam { + return reduce(source.judge.subtasks.map(s => reduce(s.cases.filter(c => c.result != null).map(c => map(c.result))))); } time = forEveryTestcase(c => c.time, _.sum); memory = forEveryTestcase(c => c.memory, _.max); const finalResult = forEveryTestcase(c => c.type, firstNonAC); statusString = statusToString[finalResult]; } + } else { + statusString = systemError; } const result = { diff --git a/src/rmq-common.ts b/src/rmq-common.ts index 5bc2399..6a53b97 100644 --- a/src/rmq-common.ts +++ b/src/rmq-common.ts @@ -1,4 +1,6 @@ import amqp = require('amqplib'); +import msgpack = require('msgpack-lite'); +import winston = require('winston'); export const maxPriority = 5; export const taskQueueName = 'task'; @@ -12,6 +14,10 @@ export async function assertTaskQueue(channel: amqp.Channel) { }); } +// Difference between result and progress: +// The `progress' is to be handled by *all* frontend proxies and pushed to all clients. +// The `result' is to be handled only *once*, and is to be written to the database. + export async function assertProgressReportExchange(channel: amqp.Channel) { await channel.assertExchange(progressExchangeName, 'fanout', { durable: false }); } @@ -25,4 +31,25 @@ export async function assertJudgeQueue(channel: amqp.Channel) { maxPriority: maxPriority, durable: true }); -} \ No newline at end of file +} + + +export async function waitForTask(conn: amqp.Connection, queueName: string, priority: number, retry: (err: Error) => boolean, handle: (task: T) => Promise) { + const channel = await conn.createChannel(); + channel.prefetch(1); + await channel.consume(queueName, (msg: amqp.Message) => { + const data = msgpack.decode(msg.content) as T; + winston.verbose('Got task', data); + + handle(data).then(async () => { + channel.ack(msg); + }, async (err) => { + if (retry) + await new Promise((res) => setTimeout(res, 300)); + winston.warn(`Failed to process message: ${err.toString()}`); + channel.nack(msg, false, retry(err)); + }); + }, { + priority: priority + }); +} diff --git a/src/runner/compile.ts b/src/runner/compile.ts index 82eb109..84e2447 100644 --- a/src/runner/compile.ts +++ b/src/runner/compile.ts @@ -4,7 +4,7 @@ import randomString = require('randomstring'); import bluebird = require('bluebird'); import getFolderSize = require('get-folder-size'); -import { CompileTask, CompileResult } from '../interfaces'; +import { CompileTask, CompilationResult, TaskStatus } from '../interfaces'; import { globalConfig as Cfg } from './config'; import { sandboxize, createOrEmptyDir, setWriteAccess } from './utils'; import { Language, getLanguage } from '../languages'; @@ -15,7 +15,7 @@ import { pushBinary } from './executable'; const getSize: any = bluebird.promisify(getFolderSize); -export async function compile(task: CompileTask): Promise { +export async function compile(task: CompileTask): Promise { const srcDir = pathLib.join(Cfg.workingDirectory, `src`); const binDir = pathLib.join(Cfg.workingDirectory, `bin`); const tempDir = pathLib.join(Cfg.workingDirectory, 'temp'); @@ -68,25 +68,25 @@ export async function compile(task: CompileTask): Promise { // If the output is too long if (outputSize > language.binarySizeLimit) { return { - status: -1, + status: TaskStatus.Failed, message: `Your source code compiled to ${outputSize} bytes which is too big, too thick, too long for us..` }; } // Else OK! } else { // If compilation error return { - status: sandboxResult.code, - message: await readFileLength(binDir + '/' + compileConfig.messageFile, Cfg.compilerMessageLimit) + status: TaskStatus.Failed, + message: await readFileLength(pathLib.join(binDir, compileConfig.messageFile), Cfg.compilerMessageLimit) }; } } else { return { - status: -1, + status: TaskStatus.Failed, message: (`A ${SandboxStatus[sandboxResult.status]} encountered while compiling your code.\n\n` + await readFileLength(binDir + '/' + compileConfig.messageFile, Cfg.compilerMessageLimit)).trim() }; } await pushBinary(task.binaryName, language, task.code, binDir); - return { status: 0 }; + return { status: TaskStatus.Done }; } finally { await Promise.all([fse.remove(binDir), fse.remove(srcDir)]); } diff --git a/src/runner/config.ts b/src/runner/config.ts index a82a418..4526262 100644 --- a/src/runner/config.ts +++ b/src/runner/config.ts @@ -1,6 +1,7 @@ -import * as commandLineArgs from 'command-line-args'; -import * as fs from 'fs'; -import * as winston from 'winston'; +import commandLineArgs = require('command-line-args'); +import fs = require('fs'); +import winston = require('winston'); +import { configureWinston } from '../winston-common'; export interface SandboxConfigBase { chroot: string; @@ -69,8 +70,4 @@ export const globalConfig: ConfigStructure = { }, } -if (options.verbose) { - (winston as any).level = 'debug'; -} else { - (winston as any).level = 'warn'; -} \ No newline at end of file +configureWinston(options.verbose); \ No newline at end of file diff --git a/src/runner/executable.ts b/src/runner/executable.ts index 03b633b..8e8fd7a 100644 --- a/src/runner/executable.ts +++ b/src/runner/executable.ts @@ -29,7 +29,7 @@ export async function pushBinary(name: string, language: Language, code: string, language: language.name, code: code }; - await putRedis(name + redisBinarySuffix, msgpack.encode(binary)); + await putRedis(name + redisBinarySuffix, binary); await putRedis(name + redisMetadataSuffix, msgpack.encode(data)); } @@ -57,6 +57,7 @@ export async function fetchBinary(name: string): Promise<[string, Language, stri winston.debug(`Doing work: fetching binary for ${name} ...`); await fse.mkdir(targetName); const binary = msgpack.decode(await getRedis(name + redisBinarySuffix)); + winston.debug(`Decompressing binary (size=${binary.length})...`); await new Promise((res, rej) => { const s = tar.extract({ cwd: targetName diff --git a/src/runner/judge.ts b/src/runner/judge.ts index af9fcf3..685a037 100644 --- a/src/runner/judge.ts +++ b/src/runner/judge.ts @@ -4,7 +4,7 @@ import fse = require('fs-extra'); import winston = require('winston'); import { SandboxStatus } from 'simple-sandbox/lib/interfaces'; -import { TaskResult, StandardRunTask, StandardRunResult } from '../interfaces'; +import { TestcaseResultType, StandardRunTask, StandardRunResult } from '../interfaces'; import { createOrEmptyDir, tryEmptyDir } from './utils'; import { readFileLength, tryReadFile } from '../utils'; import { globalConfig as Cfg } from './config'; @@ -17,7 +17,7 @@ const workingDir = `${Cfg.workingDirectory}/data`; const spjWorkingDir = `${Cfg.workingDirectory}/data-spj`; interface SpjResult { - status: TaskResult; + status: TestcaseResultType; message: string; score: number; } @@ -37,7 +37,7 @@ async function runSpj(spjBinDir: string, spjLanguage: Language): Promise spjFullScore) { return { - status: TaskResult.JudgementFailed, + status: TestcaseResultType.JudgementFailed, message: `Special Judge returned an unrecoginzed score: ${scoreString}.`, score: 0 }; } else { - let status: TaskResult; + let status: TestcaseResultType; switch (score) { case spjFullScore: - status = TaskResult.Accepted; + status = TestcaseResultType.Accepted; break; case 0: - status = TaskResult.WrongAnswer; + status = TestcaseResultType.WrongAnswer; break; default: - status = TaskResult.PartiallyCorrect; + status = TestcaseResultType.PartiallyCorrect; break; } return { @@ -131,18 +131,18 @@ export async function judgeStandard(task: StandardRunTask): Promise