From 9bf6bdb137f1d846a4031b156d05ec11312a65cc Mon Sep 17 00:00:00 2001 From: Menci Date: Fri, 26 Apr 2019 01:45:56 +0800 Subject: [PATCH] Optimize problem statistics --- libs/judger.js | 2 +- models/common.ts | 2 +- models/judge_state.ts | 16 +- models/problem.ts | 296 ++++++++++---------------------- models/submission_statistics.ts | 33 ++++ 5 files changed, 131 insertions(+), 218 deletions(-) create mode 100644 models/submission_statistics.ts diff --git a/libs/judger.js b/libs/judger.js index 0963dd1..99c3ba5 100644 --- a/libs/judger.js +++ b/libs/judger.js @@ -179,7 +179,7 @@ async function connect() { judge_state.max_memory = convertedResult.memory; judge_state.result = convertedResult.result; await judge_state.save(); - await judge_state.updateRelatedInfo(); + await judge_state.updateRelatedInfo(false); } else if (result.type == interface.ProgressReportType.Compiled) { if (!judge_state) return; judge_state.compilation = result.progress; diff --git a/models/common.ts b/models/common.ts index e533051..4317153 100644 --- a/models/common.ts +++ b/models/common.ts @@ -112,7 +112,7 @@ export default class Model extends TypeORM.BaseEntity { return await queryBuilder.getMany(); } - static async queryPage(paginater: Paginater, where, order, largeData) { + static async queryPage(paginater: Paginater, where, order, largeData = false) { if (!paginater.pageCnt) return []; const queryBuilder = where instanceof TypeORM.SelectQueryBuilder diff --git a/models/judge_state.ts b/models/judge_state.ts index fc8cdf8..1cf0404 100644 --- a/models/judge_state.ts +++ b/models/judge_state.ts @@ -12,6 +12,7 @@ const Judger = syzoj.lib('judger'); @TypeORM.Entity() @TypeORM.Index(['type', 'type_info']) @TypeORM.Index(['type', 'is_public']) +@TypeORM.Index(['problem_id', 'type', 'pending', 'score']) export default class JudgeState extends Model { @TypeORM.PrimaryGeneratedColumn() id: number; @@ -114,6 +115,10 @@ export default class JudgeState extends Model { // No need to await them. this.user.refreshSubmitInfo(); this.problem.resetSubmissionCount(); + + if (!newSubmission) { + this.problem.updateStatistics(this.user_id); + } } else if (this.type === 1) { let contest = await Contest.findById(this.type_info); await contest.newSubmission(this); @@ -138,16 +143,7 @@ export default class JudgeState extends Model { this.task_id = require('randomstring').generate(10); await this.save(); - await this.problem.resetSubmissionCount(); - if (oldStatus === 'Accepted') { - await this.user.refreshSubmitInfo(); - await this.user.save(); - } - - if (this.type === 1) { - let contest = await Contest.findById(this.type_info); - await contest.newSubmission(this); - } + await this.updateRelatedInfo(false); try { await Judger.judge(this, this.problem, 1); diff --git a/models/problem.ts b/models/problem.ts index 89f3587..c2621db 100644 --- a/models/problem.ts +++ b/models/problem.ts @@ -1,181 +1,3 @@ -const statisticsStatements = { - fastest: - '\ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `total_time` ASC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `total_time` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `total_time` ASC \ - LIMIT 1 \ - ) AS `total_time` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `total_time` ASC \ -', - slowest: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `total_time` DESC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `total_time` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `total_time` DESC \ - LIMIT 1 \ - ) AS `total_time` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `total_time` DESC \ -', - shortest: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `code_length` ASC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `code_length` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `code_length` ASC \ - LIMIT 1 \ - ) AS `code_length` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `code_length` ASC \ -', - longest: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `code_length` DESC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `code_length` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `code_length` DESC \ - LIMIT 1 \ - ) AS `code_length` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `code_length` DESC \ -', - earliest: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `submit_time` ASC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `submit_time` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `submit_time` ASC \ - LIMIT 1 \ - ) AS `submit_time` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `submit_time` ASC \ -', - min: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `max_memory` ASC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `max_memory` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `max_memory` ASC \ - LIMIT 1 \ - ) AS `max_memory` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `max_memory` ASC \ -', - max: - ' \ -SELECT \ - DISTINCT(`user_id`) AS `user_id`, \ - ( \ - SELECT \ - `id` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `max_memory` ASC \ - LIMIT 1 \ - ) AS `id`, \ - ( \ - SELECT \ - `max_memory` \ - FROM `judge_state` `inner_table` \ - WHERE `problem_id` = `outer_table`.`problem_id` AND `user_id` = `outer_table`.`user_id` AND `status` = "Accepted" AND `type` = 0 \ - ORDER BY `max_memory` ASC \ - LIMIT 1 \ - ) AS `max_memory` \ -FROM `judge_state` `outer_table` \ -WHERE \ - `problem_id` = __PROBLEM_ID__ AND `status` = "Accepted" AND `type` = 0 \ -ORDER BY `max_memory` DESC \ -' -}; - import * as TypeORM from "typeorm"; import Model from "./common"; @@ -187,6 +9,7 @@ import JudgeState from "./judge_state"; import Contest from "./contest"; import ProblemTag from "./problem_tag"; import ProblemTagMap from "./problem_tag_map"; +import SubmissionStatictics, { StatisticsType } from "./submission_statistics"; import * as fs from "fs-extra"; import * as path from "path"; @@ -204,6 +27,18 @@ enum ProblemType { Interaction = "interaction" } +const statisticsTypes = { + fastest: ['total_time', 'ASC'], + slowest: ['total_time', 'DESC'], + shortest: ['code_length', 'ASC'], + longest: ['code_length', 'DESC'], + min: ['max_memory', 'ASC'], + max: ['max_memory', 'DESC'], + earliest: ['submit_time', 'ASC'] +}; + +const statisticsCodeOnly = ["fastest", "slowest", "min", "max"]; + @TypeORM.Entity() export default class Problem extends Model { static cache = true; @@ -505,18 +340,50 @@ export default class Problem extends Model { }); } - // type: fastest / slowest / shortest / longest / earliest - async countStatistics(type) { - let statement = statisticsStatements[type]; - if (!statement) return null; + async updateStatistics(user_id) { + await Promise.all(Object.keys(statisticsTypes).map(async type => { + if (this.type === ProblemType.SubmitAnswer && statisticsCodeOnly.includes(type)) return; + + await syzoj.utils.lock(['Problem::UpdateStatistics', this.id, type], async () => { + const [column, order] = statisticsTypes[type]; + const result = await JudgeState.createQueryBuilder() + .select([column, "id"]) + .where("user_id = :user_id", { user_id }) + .andWhere("status = :status", { status: "Accepted" }) + .andWhere("problem_id = :problem_id", { problem_id: this.id }) + .orderBy({ [column]: order }) + .take(1) + .getRawMany(); + const resultRow = result[0]; + + if (!resultRow || resultRow[column] == null) return; + + const baseColumns = { + user_id, + problem_id: this.id, + type: type as StatisticsType + }; - const entityManager = TypeORM.getManager(); + let record = await SubmissionStatictics.findOne(baseColumns); + if (!record) { + record = SubmissionStatictics.create(baseColumns); + } + + record.key = resultRow[column]; + record.submission_id = resultRow["id"]; - statement = statement.replace('__PROBLEM_ID__', this.id); - return JudgeState.countQuery(statement); + await record.save(); + }); + })); + } + + async countStatistics(type) { + return await SubmissionStatictics.count({ + problem_id: this.id, + type: type + }); } - // type: fastest / slowest / shortest / longest / earliest async getStatistics(type, paginate) { const entityManager = TypeORM.getManager(); @@ -528,18 +395,29 @@ export default class Problem extends Model { suffixSum: null }; - let statement = statisticsStatements[type]; - if (!statement) return null; - - statement = statement.replace('__PROBLEM_ID__', this.id); - - let a; - if (!paginate.pageCnt) a = []; - else a = (await entityManager.query(statement + `LIMIT ${paginate.perPage} OFFSET ${(paginate.currPage - 1) * paginate.perPage}`)); - - statistics.judge_state = await a.mapAsync(async x => JudgeState.findById(x.id)); - - a = (await entityManager.query('SELECT `score`, COUNT(*) AS `count` FROM `judge_state` WHERE `problem_id` = __PROBLEM_ID__ AND `type` = 0 AND `pending` = 0 GROUP BY `score`'.replace('__PROBLEM_ID__', this.id.toString()))); + const order = statisticsTypes[type][1]; + const ids = (await SubmissionStatictics.queryPage(paginate, { + problem_id: this.id, + type: type + }, { + '`key`': order + })).map(x => x.submission_id); + + statistics.judge_state = ids.length ? await JudgeState.createQueryBuilder() + .whereInIds(ids) + .orderBy(`FIELD(id,${ids.join(',')})`) + .getMany() + : []; + + JudgeState.createQueryBuilder() + .select('score') + .addSelect('COUNT(*)', 'count') + .where('problem_id = :problem_id', { problem_id: this.id }) + .andWhere('type = 0') + .andWhere('pending = false') + .groupBy('score') + .getRawMany() + let a = (await entityManager.query('SELECT `score`, COUNT(*) AS `count` FROM `judge_state` WHERE `problem_id` = __PROBLEM_ID__ AND `type` = 0 AND `pending` = 0 GROUP BY `score`'.replace('__PROBLEM_ID__', this.id.toString()))); let scoreCount = []; for (let score of a) { @@ -549,6 +427,11 @@ export default class Problem extends Model { if (scoreCount[0] === undefined) scoreCount[0] = 0; if (scoreCount[100] === undefined) scoreCount[100] = 0; + if (a[null as any]) { + a[0] += a[null as any]; + delete a[null as any]; + } + statistics.scoreDistribution = []; for (let i = 0; i < scoreCount.length; i++) { if (scoreCount[i] !== undefined) statistics.scoreDistribution.push({ score: i, count: parseInt(scoreCount[i]) }); @@ -627,11 +510,11 @@ export default class Problem extends Model { const entityManager = TypeORM.getManager(); id = parseInt(id); - await entityManager.query('UPDATE `problem` SET `id` = ' + id + ' WHERE `id` = ' + this.id); - await entityManager.query('UPDATE `judge_state` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); - await entityManager.query('UPDATE `problem_tag_map` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); - await entityManager.query('UPDATE `article` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); - + await entityManager.query('UPDATE `problem` SET `id` = ' + id + ' WHERE `id` = ' + this.id); + await entityManager.query('UPDATE `judge_state` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); + await entityManager.query('UPDATE `problem_tag_map` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); + await entityManager.query('UPDATE `article` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); + await entityManager.query('UPDATE `SubmissionStatictics` SET `problem_id` = ' + id + ' WHERE `problem_id` = ' + this.id); let contests = await Contest.find(); for (let contest of contests) { @@ -698,9 +581,10 @@ export default class Problem extends Model { problemTagCache.del(this.id); - await entityManager.query('DELETE FROM `judge_state` WHERE `problem_id` = ' + this.id); - await entityManager.query('DELETE FROM `problem_tag_map` WHERE `problem_id` = ' + this.id); - await entityManager.query('DELETE FROM `article` WHERE `problem_id` = ' + this.id); + await entityManager.query('DELETE FROM `judge_state` WHERE `problem_id` = ' + this.id); + await entityManager.query('DELETE FROM `problem_tag_map` WHERE `problem_id` = ' + this.id); + await entityManager.query('DELETE FROM `article` WHERE `problem_id` = ' + this.id); + await entityManager.query('DELETE FROM `submission_statictics` WHERE `problem_id` = ' + this.id); await this.destroy(); } diff --git a/models/submission_statistics.ts b/models/submission_statistics.ts new file mode 100644 index 0000000..9d84936 --- /dev/null +++ b/models/submission_statistics.ts @@ -0,0 +1,33 @@ +import * as TypeORM from "typeorm"; +import Model from "./common"; + +export enum StatisticsType { + FASTEST = "fastest", + SLOWEST = "slowest", + SHORTEST = "shortest", + LONGEST = "longest", + MEMORY_MIN = "min", + MEMORY_MAX = "max", + EARLIEST = "earliest" +} + +@TypeORM.Entity() +@TypeORM.Index(['problem_id', 'type', 'key']) +export default class SubmissionStatistics extends Model { + static cache = false; + + @TypeORM.PrimaryColumn({ type: "integer" }) + problem_id: number; + + @TypeORM.PrimaryColumn({ type: "integer" }) + user_id: number; + + @TypeORM.PrimaryColumn({ type: "enum", enum: StatisticsType }) + type: StatisticsType; + + @TypeORM.Column({ type: "integer" }) + key: number; + + @TypeORM.Column({ type: "integer" }) + submission_id: number; +};