From 2563d8682b62bb5bc1881fa6c4cafb81ff9fb6f3 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:40:42 +0100 Subject: [PATCH 01/29] enforce realtime difficulty calculation when using non-default configuration on mods --- README.md | 4 +- .../Stores/BeatmapStore.cs | 363 +++++++++--------- 2 files changed, 193 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index b28435e5..a55b43a1 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ Whether to process user total stats. Set to `0` to disable processing. Default is unset (processing enabled). -### REALTIME_DIFFICULTY +### ALWAYS_REALTIME_DIFFICULTY -Whether to use realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data. Set to `0` to disable processing. +Whether to use **always** realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. Default is unset (processing enabled). diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 59d503f8..369e1bf5 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,172 +1,191 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (use_realtime_difficulty_calculation) - { - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - return calculator.Calculate(mods); - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 9d2cd8782f4120710fe6dd9175319989ab70114e Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:48:40 +0100 Subject: [PATCH 02/29] add additional support for realtime calculations on non-legacy mods --- .../Stores/BeatmapStore.cs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 369e1bf5..34e46cc4 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -69,8 +69,9 @@ public static async Task CreateAsync(MySqlConnection connection, M { // database attributes are stored using the default mod configurations // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration); + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { @@ -126,6 +127,27 @@ public static async Task CreateAsync(MySqlConnection connection, M return difficultyAttributes; } + /// + /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModRelax + or ModHalfTime + or ModFlashlight + or ModCinema + or ModAutoplay + or ModScoreV2; + /// /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. /// The match is not always exact; for some mods that award pp but do not exist in stable From 3b159d5c2c507ce9e5599b4cc2a551f1d8f0bd06 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:50:22 +0100 Subject: [PATCH 03/29] line endings --- .../Stores/BeatmapStore.cs | 426 +++++++++--------- 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 34e46cc4..68506ede 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,213 +1,213 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using StatsdClient; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - // database attributes are stored using the default mod configurations - // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database - // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); - - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) - { - var stopwatch = Stopwatch.StartNew(); - - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - var attributes = calculator.Calculate(mods); - - string[] tags = - { - $"ruleset:{ruleset.RulesetInfo.OnlineID}", - $"mods:{string.Join("", mods.Select(x => x.Acronym))}" - }; - - DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); - - return attributes; - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. - /// Used by to decide if the current mod combination's difficulty attributes - /// can be fetched from the database. - /// - private static bool isLegacyMod(Mod mod) => - mod is ModNoFail - or ModEasy - or ModHidden - or ModHardRock - or ModPerfect - or ModSuddenDeath - or ModNightcore - or ModDoubleTime - or ModRelax - or ModHalfTime - or ModFlashlight - or ModCinema - or ModAutoplay - or ModScoreV2; - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModRelax + or ModHalfTime + or ModFlashlight + or ModCinema + or ModAutoplay + or ModScoreV2; + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 435a1f5728f2dbffbd11d31dd6fcb857fab24652 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:54:37 +0100 Subject: [PATCH 04/29] remove unneeded interpolation --- .../Stores/BeatmapStore.cs | 426 +++++++++--------- 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 68506ede..d5fdc085 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,213 +1,213 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using StatsdClient; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - // database attributes are stored using the default mod configurations - // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database - // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); - - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) - { - var stopwatch = Stopwatch.StartNew(); - - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - var attributes = calculator.Calculate(mods); - - string[] tags = - { - $"ruleset:{ruleset.RulesetInfo.OnlineID}", - $"mods:{string.Join("", mods.Select(x => x.Acronym))}" - }; - - DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); - - return attributes; - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. - /// Used by to decide if the current mod combination's difficulty attributes - /// can be fetched from the database. - /// - private static bool isLegacyMod(Mod mod) => - mod is ModNoFail - or ModEasy - or ModHidden - or ModHardRock - or ModPerfect - or ModSuddenDeath - or ModNightcore - or ModDoubleTime - or ModRelax - or ModHalfTime - or ModFlashlight - or ModCinema - or ModAutoplay - or ModScoreV2; - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModRelax + or ModHalfTime + or ModFlashlight + or ModCinema + or ModAutoplay + or ModScoreV2; + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 749f505e84f91273db7136c3581d32cc54bf46a5 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 19:42:33 +0100 Subject: [PATCH 05/29] line endings --- .../Stores/BeatmapStore.cs | 426 +++++++++--------- 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index d5fdc085..73c6d49d 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,213 +1,213 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using StatsdClient; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - // database attributes are stored using the default mod configurations - // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database - // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); - - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) - { - var stopwatch = Stopwatch.StartNew(); - - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - var attributes = calculator.Calculate(mods); - - string[] tags = - { - $"ruleset:{ruleset.RulesetInfo.OnlineID}", - $"mods:{string.Join("", mods.Select(x => x.Acronym))}" - }; - - DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); - - return attributes; - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. - /// Used by to decide if the current mod combination's difficulty attributes - /// can be fetched from the database. - /// - private static bool isLegacyMod(Mod mod) => - mod is ModNoFail - or ModEasy - or ModHidden - or ModHardRock - or ModPerfect - or ModSuddenDeath - or ModNightcore - or ModDoubleTime - or ModRelax - or ModHalfTime - or ModFlashlight - or ModCinema - or ModAutoplay - or ModScoreV2; - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModRelax + or ModHalfTime + or ModFlashlight + or ModCinema + or ModAutoplay + or ModScoreV2; + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 8aae8a2804ace8307245ea1f68fe3e19bef809dc Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 19:49:59 +0100 Subject: [PATCH 06/29] fix CL edge case --- .../Stores/BeatmapStore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 73c6d49d..1186ecc3 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -69,9 +69,9 @@ public static async Task CreateAsync(MySqlConnection connection, M { // database attributes are stored using the default mod configurations // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database + // or non-legacy mods which aren't populated into the database (with exception to CL) // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isLegacyMod(m) && m is not ModClassic)); if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { From 927af4e2864336874f996f40d4eb8c98db7e8ca0 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:22:53 +0100 Subject: [PATCH 07/29] rename `isLegacyMod` to `isRankedLegacyMod` and add missing mania mods --- .../Stores/BeatmapStore.cs | 431 +++++++++--------- 1 file changed, 218 insertions(+), 213 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 1186ecc3..664e9a95 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,213 +1,218 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using StatsdClient; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - // database attributes are stored using the default mod configurations - // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database (with exception to CL) - // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isLegacyMod(m) && m is not ModClassic)); - - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) - { - var stopwatch = Stopwatch.StartNew(); - - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - var attributes = calculator.Calculate(mods); - - string[] tags = - { - $"ruleset:{ruleset.RulesetInfo.OnlineID}", - $"mods:{string.Join("", mods.Select(x => x.Acronym))}" - }; - - DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); - - return attributes; - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. - /// Used by to decide if the current mod combination's difficulty attributes - /// can be fetched from the database. - /// - private static bool isLegacyMod(Mod mod) => - mod is ModNoFail - or ModEasy - or ModHidden - or ModHardRock - or ModPerfect - or ModSuddenDeath - or ModNightcore - or ModDoubleTime - or ModRelax - or ModHalfTime - or ModFlashlight - or ModCinema - or ModAutoplay - or ModScoreV2; - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mania.Mods; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database (with exception to CL) + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to create a simple solution to deciding if a can be considered a ranked "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isRankedLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden // this also catches ManiaModFadeIn + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModHalfTime + or ModFlashlight + or ModTouchDevice + or ManiaModKey4 + or ManiaModKey5 + or ManiaModKey6 + or ManiaModKey7 + or ManiaModKey8 + or ManiaModKey9 + or ManiaModMirror; + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 59ca627363b236bd9eec1bcac13b0379b5e854d2 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:24:15 +0100 Subject: [PATCH 08/29] line endings (sigh) --- .../Stores/BeatmapStore.cs | 436 +++++++++--------- 1 file changed, 218 insertions(+), 218 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 664e9a95..dfafebce 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -1,218 +1,218 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading.Tasks; -using Dapper; -using MySqlConnector; -using osu.Framework.IO.Network; -using osu.Game.Beatmaps; -using osu.Game.Beatmaps.Legacy; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Mania.Mods; -using osu.Game.Rulesets.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using StatsdClient; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores -{ - /// - /// A store for retrieving s. - /// - public class BeatmapStore - { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; - private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; - - private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); - private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); - private readonly IReadOnlyDictionary blacklist; - - private BeatmapStore(IEnumerable> blacklist) - { - this.blacklist = new Dictionary(blacklist); - } - - /// - /// Creates a new . - /// - /// The . - /// An existing transaction. - /// The created . - public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) - { - var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); - - return new BeatmapStore - ( - dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) - ); - } - - /// - /// Retrieves difficulty attributes from the database. - /// - /// The beatmap. - /// The score's ruleset. - /// The score's mods. - /// The . - /// An existing transaction. - /// The difficulty attributes or null if not existing. - public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) - { - // database attributes are stored using the default mod configurations - // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database (with exception to CL) - // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); - - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) - { - var stopwatch = Stopwatch.StartNew(); - - using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); - - req.AllowInsecureRequests = true; - - await req.PerformAsync().ConfigureAwait(false); - - if (req.ResponseStream.Length == 0) - throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); - - var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); - var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - - var attributes = calculator.Calculate(mods); - - string[] tags = - { - $"ruleset:{ruleset.RulesetInfo.OnlineID}", - $"mods:{string.Join("", mods.Select(x => x.Acronym))}" - }; - - DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); - - return attributes; - } - - BeatmapDifficultyAttribute[]? rawDifficultyAttributes; - - LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); - DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); - - if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) - { - rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( - "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new - { - key.BeatmapId, - key.RulesetId, - key.ModValue - }, transaction: transaction)).ToArray(); - } - - if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) - return null; - - DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); - difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); - - return difficultyAttributes; - } - - /// - /// This method attempts to create a simple solution to deciding if a can be considered a ranked "legacy" mod. - /// Used by to decide if the current mod combination's difficulty attributes - /// can be fetched from the database. - /// - private static bool isRankedLegacyMod(Mod mod) => - mod is ModNoFail - or ModEasy - or ModHidden // this also catches ManiaModFadeIn - or ModHardRock - or ModPerfect - or ModSuddenDeath - or ModNightcore - or ModDoubleTime - or ModHalfTime - or ModFlashlight - or ModTouchDevice - or ManiaModKey4 - or ManiaModKey5 - or ManiaModKey6 - or ManiaModKey7 - or ManiaModKey8 - or ManiaModKey9 - or ManiaModMirror; - - /// - /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. - /// The match is not always exact; for some mods that award pp but do not exist in stable - /// (such as ) the closest available approximation is used. - /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . - /// - private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) - { - var legacyMods = ruleset.ConvertToLegacyMods(mods); - - // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) - if (mods.Any(mod => mod is ModDaycore)) - legacyMods |= LegacyMods.HalfTime; - - return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); - } - - /// - /// Retrieves a beatmap from the database. - /// - /// The beatmap's ID. - /// The . - /// An existing transaction. - /// The retrieved beatmap, or null if not existing. - public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) - { - if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) - return beatmap; - - return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new - { - BeatmapId = beatmapId - }, transaction: transaction); - } - - /// - /// Whether performance points may be awarded for the given beatmap and ruleset combination. - /// - /// The beatmap. - /// The ruleset. - public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) - { - if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) - return false; - - switch (beatmap.approved) - { - case BeatmapOnlineStatus.Ranked: - case BeatmapOnlineStatus.Approved: - return true; - - default: - return false; - } - } - - private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); - - [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] - private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using Dapper; +using MySqlConnector; +using osu.Framework.IO.Network; +using osu.Game.Beatmaps; +using osu.Game.Beatmaps.Legacy; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mania.Mods; +using osu.Game.Rulesets.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores +{ + /// + /// A store for retrieving s. + /// + public class BeatmapStore + { + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; + + private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); + private readonly ConcurrentDictionary attributeCache = new ConcurrentDictionary(); + private readonly IReadOnlyDictionary blacklist; + + private BeatmapStore(IEnumerable> blacklist) + { + this.blacklist = new Dictionary(blacklist); + } + + /// + /// Creates a new . + /// + /// The . + /// An existing transaction. + /// The created . + public static async Task CreateAsync(MySqlConnection connection, MySqlTransaction? transaction = null) + { + var dbBlacklist = await connection.QueryAsync("SELECT * FROM osu_beatmap_performance_blacklist", transaction: transaction); + + return new BeatmapStore + ( + dbBlacklist.Select(b => new KeyValuePair(new BlacklistEntry(b.beatmap_id, b.mode), 1)) + ); + } + + /// + /// Retrieves difficulty attributes from the database. + /// + /// The beatmap. + /// The score's ruleset. + /// The score's mods. + /// The . + /// An existing transaction. + /// The difficulty attributes or null if not existing. + public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) + { + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database (with exception to CL) + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + { + var stopwatch = Stopwatch.StartNew(); + + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); + + req.AllowInsecureRequests = true; + + await req.PerformAsync().ConfigureAwait(false); + + if (req.ResponseStream.Length == 0) + throw new Exception($"Retrieved zero-length beatmap ({beatmap.beatmap_id})!"); + + var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); + var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); + + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; + } + + BeatmapDifficultyAttribute[]? rawDifficultyAttributes; + + LegacyMods legacyModValue = getLegacyModsForAttributeLookup(beatmap, ruleset, mods); + DifficultyAttributeKey key = new DifficultyAttributeKey(beatmap.beatmap_id, (uint)ruleset.RulesetInfo.OnlineID, (uint)legacyModValue); + + if (!attributeCache.TryGetValue(key, out rawDifficultyAttributes)) + { + rawDifficultyAttributes = attributeCache[key] = (await connection.QueryAsync( + "SELECT * FROM osu_beatmap_difficulty_attribs WHERE `beatmap_id` = @BeatmapId AND `mode` = @RulesetId AND `mods` = @ModValue", new + { + key.BeatmapId, + key.RulesetId, + key.ModValue + }, transaction: transaction)).ToArray(); + } + + if (rawDifficultyAttributes == null || rawDifficultyAttributes.Length == 0) + return null; + + DifficultyAttributes difficultyAttributes = LegacyRulesetHelper.CreateDifficultyAttributes(ruleset.RulesetInfo.OnlineID); + difficultyAttributes.FromDatabaseAttributes(rawDifficultyAttributes.ToDictionary(a => (int)a.attrib_id, a => (double)a.value), beatmap); + + return difficultyAttributes; + } + + /// + /// This method attempts to create a simple solution to deciding if a can be considered a ranked "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isRankedLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden // this also catches ManiaModFadeIn + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModHalfTime + or ModFlashlight + or ModTouchDevice + or ManiaModKey4 + or ManiaModKey5 + or ManiaModKey6 + or ManiaModKey7 + or ManiaModKey8 + or ManiaModKey9 + or ManiaModMirror; + + /// + /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. + /// The match is not always exact; for some mods that award pp but do not exist in stable + /// (such as ) the closest available approximation is used. + /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. + /// The entirety of this workaround is not used / unnecessary if is . + /// + private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) + { + var legacyMods = ruleset.ConvertToLegacyMods(mods); + + // mods that are not represented in `LegacyMods` (but we can approximate them well enough with others) + if (mods.Any(mod => mod is ModDaycore)) + legacyMods |= LegacyMods.HalfTime; + + return LegacyModsHelper.MaskRelevantMods(legacyMods, ruleset.RulesetInfo.OnlineID != beatmap.playmode, ruleset.RulesetInfo.OnlineID); + } + + /// + /// Retrieves a beatmap from the database. + /// + /// The beatmap's ID. + /// The . + /// An existing transaction. + /// The retrieved beatmap, or null if not existing. + public async Task GetBeatmapAsync(uint beatmapId, MySqlConnection connection, MySqlTransaction? transaction = null) + { + if (beatmapCache.TryGetValue(beatmapId, out var beatmap)) + return beatmap; + + return beatmapCache[beatmapId] = await connection.QuerySingleOrDefaultAsync("SELECT * FROM osu_beatmaps WHERE `beatmap_id` = @BeatmapId", new + { + BeatmapId = beatmapId + }, transaction: transaction); + } + + /// + /// Whether performance points may be awarded for the given beatmap and ruleset combination. + /// + /// The beatmap. + /// The ruleset. + public bool IsBeatmapValidForPerformance(Beatmap beatmap, uint rulesetId) + { + if (blacklist.ContainsKey(new BlacklistEntry(beatmap.beatmap_id, rulesetId))) + return false; + + switch (beatmap.approved) + { + case BeatmapOnlineStatus.Ranked: + case BeatmapOnlineStatus.Approved: + return true; + + default: + return false; + } + } + + private record struct DifficultyAttributeKey(uint BeatmapId, uint RulesetId, uint ModValue); + + [SuppressMessage("ReSharper", "NotAccessedPositionalProperty.Local")] + private record struct BlacklistEntry(uint BeatmapId, uint RulesetId); + } +} From 9835e522231ef3d3285ab628cb0a816d0ef9a7eb Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:28:02 +0100 Subject: [PATCH 09/29] add `OsuModSpunOut` --- .../Stores/BeatmapStore.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index dfafebce..538924ba 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -17,6 +17,7 @@ using osu.Game.Rulesets.Difficulty; using osu.Game.Rulesets.Mania.Mods; using osu.Game.Rulesets.Mods; +using osu.Game.Rulesets.Osu.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; using StatsdClient; @@ -145,6 +146,7 @@ or ModDoubleTime or ModHalfTime or ModFlashlight or ModTouchDevice + or OsuModSpunOut or ManiaModKey4 or ManiaModKey5 or ManiaModKey6 From 8480823b003cb76e5228e804ac32b10a58b19992 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:32:30 +0100 Subject: [PATCH 10/29] handle hard rock being unranked for mania --- .../Stores/BeatmapStore.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 538924ba..6499d3d4 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -14,10 +14,12 @@ using osu.Game.Beatmaps; using osu.Game.Beatmaps.Legacy; using osu.Game.Rulesets; +using osu.Game.Rulesets.Catch.Mods; using osu.Game.Rulesets.Difficulty; using osu.Game.Rulesets.Mania.Mods; using osu.Game.Rulesets.Mods; using osu.Game.Rulesets.Osu.Mods; +using osu.Game.Rulesets.Taiko.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; using StatsdClient; @@ -138,7 +140,6 @@ private static bool isRankedLegacyMod(Mod mod) => mod is ModNoFail or ModEasy or ModHidden // this also catches ManiaModFadeIn - or ModHardRock or ModPerfect or ModSuddenDeath or ModNightcore @@ -146,7 +147,10 @@ or ModDoubleTime or ModHalfTime or ModFlashlight or ModTouchDevice + or OsuModHardRock or OsuModSpunOut + or TaikoModHardRock + or CatchModHardRock or ManiaModKey4 or ManiaModKey5 or ManiaModKey6 From b540e7b91dedbc74b44d7ba751ef972ce8b06ab3 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 11:44:48 +0100 Subject: [PATCH 11/29] use `ALWAYS_REALTIME_DIFFICULTY` in `DatabaseTest` --- .../DatabaseTest.cs | 584 +++++++++--------- 1 file changed, 292 insertions(+), 292 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index f597910f..4af68614 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -1,292 +1,292 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Diagnostics; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Dapper; -using Dapper.Contrib.Extensions; -using MySqlConnector; -using osu.Framework.Extensions.ExceptionExtensions; -using osu.Game.Beatmaps; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Scoring; -using osu.Game.Scoring; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using Xunit; -using Xunit.Sdk; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests -{ - [Collection("Database tests")] // Ensure all tests hitting the database are run sequentially (no parallel execution). - public abstract class DatabaseTest : IDisposable - { - protected readonly ScoreStatisticsQueueProcessor Processor; - - protected CancellationToken CancellationToken => cancellationSource.Token; - - protected const int MAX_COMBO = 1337; - - protected const int TEST_BEATMAP_ID = 1; - protected const int TEST_BEATMAP_SET_ID = 1; - protected ushort TestBuildID; - - private readonly CancellationTokenSource cancellationSource; - - private Exception? firstError; - - protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) - { - cancellationSource = Debugger.IsAttached - ? new CancellationTokenSource() - : new CancellationTokenSource(20000); - - Environment.SetEnvironmentVariable("REALTIME_DIFFICULTY", "0"); - - Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); - Processor.Error += processorOnError; - - Processor.ClearQueue(); - - using (var db = Processor.GetDatabaseConnection()) - { - // just a safety measure for now to ensure we don't hit production. since i was running on production until now. - // will throw if not on test database. - if (db.QueryFirstOrDefault("SELECT * FROM osu_counts WHERE name = 'is_production'") != null) - throw new InvalidOperationException("You are trying to do something very silly."); - - db.Execute("TRUNCATE TABLE osu_user_stats"); - db.Execute("TRUNCATE TABLE osu_user_stats_taiko"); - db.Execute("TRUNCATE TABLE osu_user_stats_fruits"); - db.Execute("TRUNCATE TABLE osu_user_stats_mania"); - db.Execute("TRUNCATE TABLE osu_user_beatmap_playcount"); - db.Execute("TRUNCATE TABLE osu_user_month_playcount"); - db.Execute("TRUNCATE TABLE osu_beatmaps"); - db.Execute("TRUNCATE TABLE osu_beatmapsets"); - db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); - - // These tables are still views for now (todo osu-web plz). - db.Execute("DELETE FROM scores"); - db.Execute("DELETE FROM score_process_history"); - - db.Execute("TRUNCATE TABLE osu_builds"); - db.Execute("REPLACE INTO osu_counts (name, count) VALUES ('playcount', 0)"); - - TestBuildID = db.QuerySingle("INSERT INTO osu_builds (version, allow_performance) VALUES ('1.0.0', 1); SELECT LAST_INSERT_ID();"); - - db.Execute("TRUNCATE TABLE `osu_user_performance_rank`"); - db.Execute("TRUNCATE TABLE `osu_user_performance_rank_highest`"); - } - - Task.Run(() => Processor.Run(CancellationToken), CancellationToken); - } - - protected ScoreItem SetScoreForBeatmap(uint beatmapId, Action? scoreSetup = null) - { - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - { - var score = CreateTestScore(beatmapId: beatmapId); - - scoreSetup?.Invoke(score); - - conn.Execute("INSERT INTO `scores` (`id`, `user_id`, `ruleset_id`, `beatmap_id`, `has_replay`, `preserve`, `ranked`, " - + "`rank`, `passed`, `accuracy`, `max_combo`, `total_score`, `data`, `pp`, `legacy_score_id`, `legacy_total_score`, " - + "`started_at`, `ended_at`, `build_id`) " - + "VALUES (@id, @user_id, @ruleset_id, @beatmap_id, @has_replay, @preserve, @ranked, " - + "@rank, @passed, @accuracy, @max_combo, @total_score, @data, @pp, @legacy_score_id, @legacy_total_score," - + "@started_at, @ended_at, @build_id)", - new - { - score.Score.id, - score.Score.user_id, - score.Score.ruleset_id, - score.Score.beatmap_id, - score.Score.has_replay, - score.Score.preserve, - score.Score.ranked, - rank = score.Score.rank.ToString(), - score.Score.passed, - score.Score.accuracy, - score.Score.max_combo, - score.Score.total_score, - score.Score.data, - score.Score.pp, - score.Score.legacy_score_id, - score.Score.legacy_total_score, - score.Score.started_at, - score.Score.ended_at, - score.Score.build_id, - }); - PushToQueueAndWaitForProcess(score); - - return score; - } - } - - private static ulong scoreIDSource; - - protected void PushToQueueAndWaitForProcess(ScoreItem item) - { - // To keep the flow of tests simple, require single-file addition of items. - if (Processor.GetQueueSize() > 0) - throw new InvalidOperationException("Queue was still processing an item when attempting to push another one."); - - long processedBefore = Processor.TotalProcessed; - - Processor.PushToQueue(item); - - WaitForDatabaseState($"SELECT score_id FROM score_process_history WHERE score_id = {item.Score.id}", item.Score.id, CancellationToken); - WaitForTotalProcessed(processedBefore + 1, CancellationToken); - } - - public static ScoreItem CreateTestScore(uint? rulesetId = null, uint? beatmapId = null) - { - var row = new SoloScore - { - id = Interlocked.Increment(ref scoreIDSource), - user_id = 2, - beatmap_id = beatmapId ?? TEST_BEATMAP_ID, - ruleset_id = (ushort)(rulesetId ?? 0), - started_at = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(180), - ended_at = DateTimeOffset.UtcNow, - max_combo = MAX_COMBO, - total_score = 100000, - rank = ScoreRank.S, - passed = true - }; - - var scoreData = new SoloScoreData - { - Statistics = - { - { HitResult.Perfect, 5 }, - { HitResult.LargeBonus, 0 } - }, - MaximumStatistics = - { - { HitResult.Perfect, 5 }, - { HitResult.LargeBonus, 2 } - }, - }; - - row.ScoreData = scoreData; - - return new ScoreItem(row); - } - - protected void IgnoreProcessorExceptions() - { - Processor.Error -= processorOnError; - } - - protected Beatmap AddBeatmap(Action? beatmapSetup = null, Action? beatmapSetSetup = null) - { - var beatmap = new Beatmap { approved = BeatmapOnlineStatus.Ranked }; - var beatmapSet = new BeatmapSet { approved = BeatmapOnlineStatus.Ranked }; - - beatmapSetup?.Invoke(beatmap); - beatmapSetSetup?.Invoke(beatmapSet); - - if (beatmap.beatmap_id == 0) beatmap.beatmap_id = TEST_BEATMAP_ID; - if (beatmapSet.beatmapset_id == 0) beatmapSet.beatmapset_id = TEST_BEATMAP_SET_ID; - - if (beatmap.beatmapset_id > 0 && beatmap.beatmapset_id != beatmapSet.beatmapset_id) - throw new ArgumentException($"{nameof(beatmapSetup)} method specified different {nameof(beatmap.beatmapset_id)} from the one specified in the {nameof(beatmapSetSetup)} method."); - - // Copy over set ID for cases where the setup steps only set it on the beatmapSet. - beatmap.beatmapset_id = (uint)beatmapSet.beatmapset_id; - - using (var db = Processor.GetDatabaseConnection()) - { - db.Insert(beatmap); - if (db.QuerySingleOrDefault("SELECT COUNT(1) FROM `osu_beatmapsets` WHERE `beatmapset_id` = @beatmapSetId", new { beatmapSetId = beatmapSet.beatmapset_id }) == 0) - db.Insert(beatmapSet); - } - - return beatmap; - } - - protected void AddBeatmapAttributes(uint? beatmapId = null, Action? setup = null, ushort mode = 0) - where TDifficultyAttributes : DifficultyAttributes, new() - { - var attribs = new TDifficultyAttributes - { - StarRating = 5, - MaxCombo = 5, - }; - - setup?.Invoke(attribs); - - var rulesetStore = new AssemblyRulesetStore(); - var rulesetInfo = rulesetStore.GetRuleset(mode)!; - var ruleset = rulesetInfo.CreateInstance(); - - using (var db = Processor.GetDatabaseConnection()) - { - foreach (var a in attribs.ToDatabaseAttributes()) - { - db.Insert(new BeatmapDifficultyAttribute - { - beatmap_id = beatmapId ?? TEST_BEATMAP_ID, - mode = mode, - mods = (uint)ruleset.ConvertToLegacyMods(attribs.Mods), - attrib_id = (ushort)a.attributeId, - value = Convert.ToSingle(a.value), - }); - } - } - } - - protected void WaitForTotalProcessed(long count, CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - if (Processor.TotalProcessed == count) - return; - - Thread.Sleep(50); - } - - throw new XunitException("All scores were not successfully processed"); - } - - protected void WaitForDatabaseState(string sql, T expected, CancellationToken cancellationToken, object? param = null) - { - using (var db = Processor.GetDatabaseConnection()) - { - T? lastValue = default; - - while (true) - { - if (!Debugger.IsAttached) - { - if (cancellationToken.IsCancellationRequested) - throw new TimeoutException($"Waiting for database state took too long (expected: {expected} last: {lastValue} sql: {sql})"); - } - - lastValue = db.QueryFirstOrDefault(sql, param); - - if ((expected == null && lastValue == null) || expected?.Equals(lastValue) == true) - return; - - firstError?.Rethrow(); - - Thread.Sleep(50); - } - } - } - - private void processorOnError(Exception? exception, ScoreItem _) => firstError ??= exception; - -#pragma warning disable CA1816 - public virtual void Dispose() -#pragma warning restore CA1816 - { - cancellationSource.Cancel(); - } - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Dapper; +using Dapper.Contrib.Extensions; +using MySqlConnector; +using osu.Framework.Extensions.ExceptionExtensions; +using osu.Game.Beatmaps; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Scoring; +using osu.Game.Scoring; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using Xunit; +using Xunit.Sdk; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests +{ + [Collection("Database tests")] // Ensure all tests hitting the database are run sequentially (no parallel execution). + public abstract class DatabaseTest : IDisposable + { + protected readonly ScoreStatisticsQueueProcessor Processor; + + protected CancellationToken CancellationToken => cancellationSource.Token; + + protected const int MAX_COMBO = 1337; + + protected const int TEST_BEATMAP_ID = 1; + protected const int TEST_BEATMAP_SET_ID = 1; + protected ushort TestBuildID; + + private readonly CancellationTokenSource cancellationSource; + + private Exception? firstError; + + protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) + { + cancellationSource = Debugger.IsAttached + ? new CancellationTokenSource() + : new CancellationTokenSource(20000); + + Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); + + Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); + Processor.Error += processorOnError; + + Processor.ClearQueue(); + + using (var db = Processor.GetDatabaseConnection()) + { + // just a safety measure for now to ensure we don't hit production. since i was running on production until now. + // will throw if not on test database. + if (db.QueryFirstOrDefault("SELECT * FROM osu_counts WHERE name = 'is_production'") != null) + throw new InvalidOperationException("You are trying to do something very silly."); + + db.Execute("TRUNCATE TABLE osu_user_stats"); + db.Execute("TRUNCATE TABLE osu_user_stats_taiko"); + db.Execute("TRUNCATE TABLE osu_user_stats_fruits"); + db.Execute("TRUNCATE TABLE osu_user_stats_mania"); + db.Execute("TRUNCATE TABLE osu_user_beatmap_playcount"); + db.Execute("TRUNCATE TABLE osu_user_month_playcount"); + db.Execute("TRUNCATE TABLE osu_beatmaps"); + db.Execute("TRUNCATE TABLE osu_beatmapsets"); + db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); + + // These tables are still views for now (todo osu-web plz). + db.Execute("DELETE FROM scores"); + db.Execute("DELETE FROM score_process_history"); + + db.Execute("TRUNCATE TABLE osu_builds"); + db.Execute("REPLACE INTO osu_counts (name, count) VALUES ('playcount', 0)"); + + TestBuildID = db.QuerySingle("INSERT INTO osu_builds (version, allow_performance) VALUES ('1.0.0', 1); SELECT LAST_INSERT_ID();"); + + db.Execute("TRUNCATE TABLE `osu_user_performance_rank`"); + db.Execute("TRUNCATE TABLE `osu_user_performance_rank_highest`"); + } + + Task.Run(() => Processor.Run(CancellationToken), CancellationToken); + } + + protected ScoreItem SetScoreForBeatmap(uint beatmapId, Action? scoreSetup = null) + { + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + var score = CreateTestScore(beatmapId: beatmapId); + + scoreSetup?.Invoke(score); + + conn.Execute("INSERT INTO `scores` (`id`, `user_id`, `ruleset_id`, `beatmap_id`, `has_replay`, `preserve`, `ranked`, " + + "`rank`, `passed`, `accuracy`, `max_combo`, `total_score`, `data`, `pp`, `legacy_score_id`, `legacy_total_score`, " + + "`started_at`, `ended_at`, `build_id`) " + + "VALUES (@id, @user_id, @ruleset_id, @beatmap_id, @has_replay, @preserve, @ranked, " + + "@rank, @passed, @accuracy, @max_combo, @total_score, @data, @pp, @legacy_score_id, @legacy_total_score," + + "@started_at, @ended_at, @build_id)", + new + { + score.Score.id, + score.Score.user_id, + score.Score.ruleset_id, + score.Score.beatmap_id, + score.Score.has_replay, + score.Score.preserve, + score.Score.ranked, + rank = score.Score.rank.ToString(), + score.Score.passed, + score.Score.accuracy, + score.Score.max_combo, + score.Score.total_score, + score.Score.data, + score.Score.pp, + score.Score.legacy_score_id, + score.Score.legacy_total_score, + score.Score.started_at, + score.Score.ended_at, + score.Score.build_id, + }); + PushToQueueAndWaitForProcess(score); + + return score; + } + } + + private static ulong scoreIDSource; + + protected void PushToQueueAndWaitForProcess(ScoreItem item) + { + // To keep the flow of tests simple, require single-file addition of items. + if (Processor.GetQueueSize() > 0) + throw new InvalidOperationException("Queue was still processing an item when attempting to push another one."); + + long processedBefore = Processor.TotalProcessed; + + Processor.PushToQueue(item); + + WaitForDatabaseState($"SELECT score_id FROM score_process_history WHERE score_id = {item.Score.id}", item.Score.id, CancellationToken); + WaitForTotalProcessed(processedBefore + 1, CancellationToken); + } + + public static ScoreItem CreateTestScore(uint? rulesetId = null, uint? beatmapId = null) + { + var row = new SoloScore + { + id = Interlocked.Increment(ref scoreIDSource), + user_id = 2, + beatmap_id = beatmapId ?? TEST_BEATMAP_ID, + ruleset_id = (ushort)(rulesetId ?? 0), + started_at = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(180), + ended_at = DateTimeOffset.UtcNow, + max_combo = MAX_COMBO, + total_score = 100000, + rank = ScoreRank.S, + passed = true + }; + + var scoreData = new SoloScoreData + { + Statistics = + { + { HitResult.Perfect, 5 }, + { HitResult.LargeBonus, 0 } + }, + MaximumStatistics = + { + { HitResult.Perfect, 5 }, + { HitResult.LargeBonus, 2 } + }, + }; + + row.ScoreData = scoreData; + + return new ScoreItem(row); + } + + protected void IgnoreProcessorExceptions() + { + Processor.Error -= processorOnError; + } + + protected Beatmap AddBeatmap(Action? beatmapSetup = null, Action? beatmapSetSetup = null) + { + var beatmap = new Beatmap { approved = BeatmapOnlineStatus.Ranked }; + var beatmapSet = new BeatmapSet { approved = BeatmapOnlineStatus.Ranked }; + + beatmapSetup?.Invoke(beatmap); + beatmapSetSetup?.Invoke(beatmapSet); + + if (beatmap.beatmap_id == 0) beatmap.beatmap_id = TEST_BEATMAP_ID; + if (beatmapSet.beatmapset_id == 0) beatmapSet.beatmapset_id = TEST_BEATMAP_SET_ID; + + if (beatmap.beatmapset_id > 0 && beatmap.beatmapset_id != beatmapSet.beatmapset_id) + throw new ArgumentException($"{nameof(beatmapSetup)} method specified different {nameof(beatmap.beatmapset_id)} from the one specified in the {nameof(beatmapSetSetup)} method."); + + // Copy over set ID for cases where the setup steps only set it on the beatmapSet. + beatmap.beatmapset_id = (uint)beatmapSet.beatmapset_id; + + using (var db = Processor.GetDatabaseConnection()) + { + db.Insert(beatmap); + if (db.QuerySingleOrDefault("SELECT COUNT(1) FROM `osu_beatmapsets` WHERE `beatmapset_id` = @beatmapSetId", new { beatmapSetId = beatmapSet.beatmapset_id }) == 0) + db.Insert(beatmapSet); + } + + return beatmap; + } + + protected void AddBeatmapAttributes(uint? beatmapId = null, Action? setup = null, ushort mode = 0) + where TDifficultyAttributes : DifficultyAttributes, new() + { + var attribs = new TDifficultyAttributes + { + StarRating = 5, + MaxCombo = 5, + }; + + setup?.Invoke(attribs); + + var rulesetStore = new AssemblyRulesetStore(); + var rulesetInfo = rulesetStore.GetRuleset(mode)!; + var ruleset = rulesetInfo.CreateInstance(); + + using (var db = Processor.GetDatabaseConnection()) + { + foreach (var a in attribs.ToDatabaseAttributes()) + { + db.Insert(new BeatmapDifficultyAttribute + { + beatmap_id = beatmapId ?? TEST_BEATMAP_ID, + mode = mode, + mods = (uint)ruleset.ConvertToLegacyMods(attribs.Mods), + attrib_id = (ushort)a.attributeId, + value = Convert.ToSingle(a.value), + }); + } + } + } + + protected void WaitForTotalProcessed(long count, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + if (Processor.TotalProcessed == count) + return; + + Thread.Sleep(50); + } + + throw new XunitException("All scores were not successfully processed"); + } + + protected void WaitForDatabaseState(string sql, T expected, CancellationToken cancellationToken, object? param = null) + { + using (var db = Processor.GetDatabaseConnection()) + { + T? lastValue = default; + + while (true) + { + if (!Debugger.IsAttached) + { + if (cancellationToken.IsCancellationRequested) + throw new TimeoutException($"Waiting for database state took too long (expected: {expected} last: {lastValue} sql: {sql})"); + } + + lastValue = db.QueryFirstOrDefault(sql, param); + + if ((expected == null && lastValue == null) || expected?.Equals(lastValue) == true) + return; + + firstError?.Rethrow(); + + Thread.Sleep(50); + } + } + } + + private void processorOnError(Exception? exception, ScoreItem _) => firstError ??= exception; + +#pragma warning disable CA1816 + public virtual void Dispose() +#pragma warning restore CA1816 + { + cancellationSource.Cancel(); + } + } +} From 5e1b1d99d3d8da4f0feec20283364258b419bd9d Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 17:25:52 +0100 Subject: [PATCH 12/29] line endings --- .../DatabaseTest.cs | 584 +++++++++--------- 1 file changed, 292 insertions(+), 292 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index 4af68614..35a4a819 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -1,292 +1,292 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System; -using System.Diagnostics; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using Dapper; -using Dapper.Contrib.Extensions; -using MySqlConnector; -using osu.Framework.Extensions.ExceptionExtensions; -using osu.Game.Beatmaps; -using osu.Game.Rulesets; -using osu.Game.Rulesets.Difficulty; -using osu.Game.Rulesets.Scoring; -using osu.Game.Scoring; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using Xunit; -using Xunit.Sdk; -using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests -{ - [Collection("Database tests")] // Ensure all tests hitting the database are run sequentially (no parallel execution). - public abstract class DatabaseTest : IDisposable - { - protected readonly ScoreStatisticsQueueProcessor Processor; - - protected CancellationToken CancellationToken => cancellationSource.Token; - - protected const int MAX_COMBO = 1337; - - protected const int TEST_BEATMAP_ID = 1; - protected const int TEST_BEATMAP_SET_ID = 1; - protected ushort TestBuildID; - - private readonly CancellationTokenSource cancellationSource; - - private Exception? firstError; - - protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) - { - cancellationSource = Debugger.IsAttached - ? new CancellationTokenSource() - : new CancellationTokenSource(20000); - - Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); - - Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); - Processor.Error += processorOnError; - - Processor.ClearQueue(); - - using (var db = Processor.GetDatabaseConnection()) - { - // just a safety measure for now to ensure we don't hit production. since i was running on production until now. - // will throw if not on test database. - if (db.QueryFirstOrDefault("SELECT * FROM osu_counts WHERE name = 'is_production'") != null) - throw new InvalidOperationException("You are trying to do something very silly."); - - db.Execute("TRUNCATE TABLE osu_user_stats"); - db.Execute("TRUNCATE TABLE osu_user_stats_taiko"); - db.Execute("TRUNCATE TABLE osu_user_stats_fruits"); - db.Execute("TRUNCATE TABLE osu_user_stats_mania"); - db.Execute("TRUNCATE TABLE osu_user_beatmap_playcount"); - db.Execute("TRUNCATE TABLE osu_user_month_playcount"); - db.Execute("TRUNCATE TABLE osu_beatmaps"); - db.Execute("TRUNCATE TABLE osu_beatmapsets"); - db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); - - // These tables are still views for now (todo osu-web plz). - db.Execute("DELETE FROM scores"); - db.Execute("DELETE FROM score_process_history"); - - db.Execute("TRUNCATE TABLE osu_builds"); - db.Execute("REPLACE INTO osu_counts (name, count) VALUES ('playcount', 0)"); - - TestBuildID = db.QuerySingle("INSERT INTO osu_builds (version, allow_performance) VALUES ('1.0.0', 1); SELECT LAST_INSERT_ID();"); - - db.Execute("TRUNCATE TABLE `osu_user_performance_rank`"); - db.Execute("TRUNCATE TABLE `osu_user_performance_rank_highest`"); - } - - Task.Run(() => Processor.Run(CancellationToken), CancellationToken); - } - - protected ScoreItem SetScoreForBeatmap(uint beatmapId, Action? scoreSetup = null) - { - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - { - var score = CreateTestScore(beatmapId: beatmapId); - - scoreSetup?.Invoke(score); - - conn.Execute("INSERT INTO `scores` (`id`, `user_id`, `ruleset_id`, `beatmap_id`, `has_replay`, `preserve`, `ranked`, " - + "`rank`, `passed`, `accuracy`, `max_combo`, `total_score`, `data`, `pp`, `legacy_score_id`, `legacy_total_score`, " - + "`started_at`, `ended_at`, `build_id`) " - + "VALUES (@id, @user_id, @ruleset_id, @beatmap_id, @has_replay, @preserve, @ranked, " - + "@rank, @passed, @accuracy, @max_combo, @total_score, @data, @pp, @legacy_score_id, @legacy_total_score," - + "@started_at, @ended_at, @build_id)", - new - { - score.Score.id, - score.Score.user_id, - score.Score.ruleset_id, - score.Score.beatmap_id, - score.Score.has_replay, - score.Score.preserve, - score.Score.ranked, - rank = score.Score.rank.ToString(), - score.Score.passed, - score.Score.accuracy, - score.Score.max_combo, - score.Score.total_score, - score.Score.data, - score.Score.pp, - score.Score.legacy_score_id, - score.Score.legacy_total_score, - score.Score.started_at, - score.Score.ended_at, - score.Score.build_id, - }); - PushToQueueAndWaitForProcess(score); - - return score; - } - } - - private static ulong scoreIDSource; - - protected void PushToQueueAndWaitForProcess(ScoreItem item) - { - // To keep the flow of tests simple, require single-file addition of items. - if (Processor.GetQueueSize() > 0) - throw new InvalidOperationException("Queue was still processing an item when attempting to push another one."); - - long processedBefore = Processor.TotalProcessed; - - Processor.PushToQueue(item); - - WaitForDatabaseState($"SELECT score_id FROM score_process_history WHERE score_id = {item.Score.id}", item.Score.id, CancellationToken); - WaitForTotalProcessed(processedBefore + 1, CancellationToken); - } - - public static ScoreItem CreateTestScore(uint? rulesetId = null, uint? beatmapId = null) - { - var row = new SoloScore - { - id = Interlocked.Increment(ref scoreIDSource), - user_id = 2, - beatmap_id = beatmapId ?? TEST_BEATMAP_ID, - ruleset_id = (ushort)(rulesetId ?? 0), - started_at = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(180), - ended_at = DateTimeOffset.UtcNow, - max_combo = MAX_COMBO, - total_score = 100000, - rank = ScoreRank.S, - passed = true - }; - - var scoreData = new SoloScoreData - { - Statistics = - { - { HitResult.Perfect, 5 }, - { HitResult.LargeBonus, 0 } - }, - MaximumStatistics = - { - { HitResult.Perfect, 5 }, - { HitResult.LargeBonus, 2 } - }, - }; - - row.ScoreData = scoreData; - - return new ScoreItem(row); - } - - protected void IgnoreProcessorExceptions() - { - Processor.Error -= processorOnError; - } - - protected Beatmap AddBeatmap(Action? beatmapSetup = null, Action? beatmapSetSetup = null) - { - var beatmap = new Beatmap { approved = BeatmapOnlineStatus.Ranked }; - var beatmapSet = new BeatmapSet { approved = BeatmapOnlineStatus.Ranked }; - - beatmapSetup?.Invoke(beatmap); - beatmapSetSetup?.Invoke(beatmapSet); - - if (beatmap.beatmap_id == 0) beatmap.beatmap_id = TEST_BEATMAP_ID; - if (beatmapSet.beatmapset_id == 0) beatmapSet.beatmapset_id = TEST_BEATMAP_SET_ID; - - if (beatmap.beatmapset_id > 0 && beatmap.beatmapset_id != beatmapSet.beatmapset_id) - throw new ArgumentException($"{nameof(beatmapSetup)} method specified different {nameof(beatmap.beatmapset_id)} from the one specified in the {nameof(beatmapSetSetup)} method."); - - // Copy over set ID for cases where the setup steps only set it on the beatmapSet. - beatmap.beatmapset_id = (uint)beatmapSet.beatmapset_id; - - using (var db = Processor.GetDatabaseConnection()) - { - db.Insert(beatmap); - if (db.QuerySingleOrDefault("SELECT COUNT(1) FROM `osu_beatmapsets` WHERE `beatmapset_id` = @beatmapSetId", new { beatmapSetId = beatmapSet.beatmapset_id }) == 0) - db.Insert(beatmapSet); - } - - return beatmap; - } - - protected void AddBeatmapAttributes(uint? beatmapId = null, Action? setup = null, ushort mode = 0) - where TDifficultyAttributes : DifficultyAttributes, new() - { - var attribs = new TDifficultyAttributes - { - StarRating = 5, - MaxCombo = 5, - }; - - setup?.Invoke(attribs); - - var rulesetStore = new AssemblyRulesetStore(); - var rulesetInfo = rulesetStore.GetRuleset(mode)!; - var ruleset = rulesetInfo.CreateInstance(); - - using (var db = Processor.GetDatabaseConnection()) - { - foreach (var a in attribs.ToDatabaseAttributes()) - { - db.Insert(new BeatmapDifficultyAttribute - { - beatmap_id = beatmapId ?? TEST_BEATMAP_ID, - mode = mode, - mods = (uint)ruleset.ConvertToLegacyMods(attribs.Mods), - attrib_id = (ushort)a.attributeId, - value = Convert.ToSingle(a.value), - }); - } - } - } - - protected void WaitForTotalProcessed(long count, CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - if (Processor.TotalProcessed == count) - return; - - Thread.Sleep(50); - } - - throw new XunitException("All scores were not successfully processed"); - } - - protected void WaitForDatabaseState(string sql, T expected, CancellationToken cancellationToken, object? param = null) - { - using (var db = Processor.GetDatabaseConnection()) - { - T? lastValue = default; - - while (true) - { - if (!Debugger.IsAttached) - { - if (cancellationToken.IsCancellationRequested) - throw new TimeoutException($"Waiting for database state took too long (expected: {expected} last: {lastValue} sql: {sql})"); - } - - lastValue = db.QueryFirstOrDefault(sql, param); - - if ((expected == null && lastValue == null) || expected?.Equals(lastValue) == true) - return; - - firstError?.Rethrow(); - - Thread.Sleep(50); - } - } - } - - private void processorOnError(Exception? exception, ScoreItem _) => firstError ??= exception; - -#pragma warning disable CA1816 - public virtual void Dispose() -#pragma warning restore CA1816 - { - cancellationSource.Cancel(); - } - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Dapper; +using Dapper.Contrib.Extensions; +using MySqlConnector; +using osu.Framework.Extensions.ExceptionExtensions; +using osu.Game.Beatmaps; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Scoring; +using osu.Game.Scoring; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using Xunit; +using Xunit.Sdk; +using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests +{ + [Collection("Database tests")] // Ensure all tests hitting the database are run sequentially (no parallel execution). + public abstract class DatabaseTest : IDisposable + { + protected readonly ScoreStatisticsQueueProcessor Processor; + + protected CancellationToken CancellationToken => cancellationSource.Token; + + protected const int MAX_COMBO = 1337; + + protected const int TEST_BEATMAP_ID = 1; + protected const int TEST_BEATMAP_SET_ID = 1; + protected ushort TestBuildID; + + private readonly CancellationTokenSource cancellationSource; + + private Exception? firstError; + + protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) + { + cancellationSource = Debugger.IsAttached + ? new CancellationTokenSource() + : new CancellationTokenSource(20000); + + Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); + + Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); + Processor.Error += processorOnError; + + Processor.ClearQueue(); + + using (var db = Processor.GetDatabaseConnection()) + { + // just a safety measure for now to ensure we don't hit production. since i was running on production until now. + // will throw if not on test database. + if (db.QueryFirstOrDefault("SELECT * FROM osu_counts WHERE name = 'is_production'") != null) + throw new InvalidOperationException("You are trying to do something very silly."); + + db.Execute("TRUNCATE TABLE osu_user_stats"); + db.Execute("TRUNCATE TABLE osu_user_stats_taiko"); + db.Execute("TRUNCATE TABLE osu_user_stats_fruits"); + db.Execute("TRUNCATE TABLE osu_user_stats_mania"); + db.Execute("TRUNCATE TABLE osu_user_beatmap_playcount"); + db.Execute("TRUNCATE TABLE osu_user_month_playcount"); + db.Execute("TRUNCATE TABLE osu_beatmaps"); + db.Execute("TRUNCATE TABLE osu_beatmapsets"); + db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); + + // These tables are still views for now (todo osu-web plz). + db.Execute("DELETE FROM scores"); + db.Execute("DELETE FROM score_process_history"); + + db.Execute("TRUNCATE TABLE osu_builds"); + db.Execute("REPLACE INTO osu_counts (name, count) VALUES ('playcount', 0)"); + + TestBuildID = db.QuerySingle("INSERT INTO osu_builds (version, allow_performance) VALUES ('1.0.0', 1); SELECT LAST_INSERT_ID();"); + + db.Execute("TRUNCATE TABLE `osu_user_performance_rank`"); + db.Execute("TRUNCATE TABLE `osu_user_performance_rank_highest`"); + } + + Task.Run(() => Processor.Run(CancellationToken), CancellationToken); + } + + protected ScoreItem SetScoreForBeatmap(uint beatmapId, Action? scoreSetup = null) + { + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + var score = CreateTestScore(beatmapId: beatmapId); + + scoreSetup?.Invoke(score); + + conn.Execute("INSERT INTO `scores` (`id`, `user_id`, `ruleset_id`, `beatmap_id`, `has_replay`, `preserve`, `ranked`, " + + "`rank`, `passed`, `accuracy`, `max_combo`, `total_score`, `data`, `pp`, `legacy_score_id`, `legacy_total_score`, " + + "`started_at`, `ended_at`, `build_id`) " + + "VALUES (@id, @user_id, @ruleset_id, @beatmap_id, @has_replay, @preserve, @ranked, " + + "@rank, @passed, @accuracy, @max_combo, @total_score, @data, @pp, @legacy_score_id, @legacy_total_score," + + "@started_at, @ended_at, @build_id)", + new + { + score.Score.id, + score.Score.user_id, + score.Score.ruleset_id, + score.Score.beatmap_id, + score.Score.has_replay, + score.Score.preserve, + score.Score.ranked, + rank = score.Score.rank.ToString(), + score.Score.passed, + score.Score.accuracy, + score.Score.max_combo, + score.Score.total_score, + score.Score.data, + score.Score.pp, + score.Score.legacy_score_id, + score.Score.legacy_total_score, + score.Score.started_at, + score.Score.ended_at, + score.Score.build_id, + }); + PushToQueueAndWaitForProcess(score); + + return score; + } + } + + private static ulong scoreIDSource; + + protected void PushToQueueAndWaitForProcess(ScoreItem item) + { + // To keep the flow of tests simple, require single-file addition of items. + if (Processor.GetQueueSize() > 0) + throw new InvalidOperationException("Queue was still processing an item when attempting to push another one."); + + long processedBefore = Processor.TotalProcessed; + + Processor.PushToQueue(item); + + WaitForDatabaseState($"SELECT score_id FROM score_process_history WHERE score_id = {item.Score.id}", item.Score.id, CancellationToken); + WaitForTotalProcessed(processedBefore + 1, CancellationToken); + } + + public static ScoreItem CreateTestScore(uint? rulesetId = null, uint? beatmapId = null) + { + var row = new SoloScore + { + id = Interlocked.Increment(ref scoreIDSource), + user_id = 2, + beatmap_id = beatmapId ?? TEST_BEATMAP_ID, + ruleset_id = (ushort)(rulesetId ?? 0), + started_at = DateTimeOffset.UtcNow - TimeSpan.FromSeconds(180), + ended_at = DateTimeOffset.UtcNow, + max_combo = MAX_COMBO, + total_score = 100000, + rank = ScoreRank.S, + passed = true + }; + + var scoreData = new SoloScoreData + { + Statistics = + { + { HitResult.Perfect, 5 }, + { HitResult.LargeBonus, 0 } + }, + MaximumStatistics = + { + { HitResult.Perfect, 5 }, + { HitResult.LargeBonus, 2 } + }, + }; + + row.ScoreData = scoreData; + + return new ScoreItem(row); + } + + protected void IgnoreProcessorExceptions() + { + Processor.Error -= processorOnError; + } + + protected Beatmap AddBeatmap(Action? beatmapSetup = null, Action? beatmapSetSetup = null) + { + var beatmap = new Beatmap { approved = BeatmapOnlineStatus.Ranked }; + var beatmapSet = new BeatmapSet { approved = BeatmapOnlineStatus.Ranked }; + + beatmapSetup?.Invoke(beatmap); + beatmapSetSetup?.Invoke(beatmapSet); + + if (beatmap.beatmap_id == 0) beatmap.beatmap_id = TEST_BEATMAP_ID; + if (beatmapSet.beatmapset_id == 0) beatmapSet.beatmapset_id = TEST_BEATMAP_SET_ID; + + if (beatmap.beatmapset_id > 0 && beatmap.beatmapset_id != beatmapSet.beatmapset_id) + throw new ArgumentException($"{nameof(beatmapSetup)} method specified different {nameof(beatmap.beatmapset_id)} from the one specified in the {nameof(beatmapSetSetup)} method."); + + // Copy over set ID for cases where the setup steps only set it on the beatmapSet. + beatmap.beatmapset_id = (uint)beatmapSet.beatmapset_id; + + using (var db = Processor.GetDatabaseConnection()) + { + db.Insert(beatmap); + if (db.QuerySingleOrDefault("SELECT COUNT(1) FROM `osu_beatmapsets` WHERE `beatmapset_id` = @beatmapSetId", new { beatmapSetId = beatmapSet.beatmapset_id }) == 0) + db.Insert(beatmapSet); + } + + return beatmap; + } + + protected void AddBeatmapAttributes(uint? beatmapId = null, Action? setup = null, ushort mode = 0) + where TDifficultyAttributes : DifficultyAttributes, new() + { + var attribs = new TDifficultyAttributes + { + StarRating = 5, + MaxCombo = 5, + }; + + setup?.Invoke(attribs); + + var rulesetStore = new AssemblyRulesetStore(); + var rulesetInfo = rulesetStore.GetRuleset(mode)!; + var ruleset = rulesetInfo.CreateInstance(); + + using (var db = Processor.GetDatabaseConnection()) + { + foreach (var a in attribs.ToDatabaseAttributes()) + { + db.Insert(new BeatmapDifficultyAttribute + { + beatmap_id = beatmapId ?? TEST_BEATMAP_ID, + mode = mode, + mods = (uint)ruleset.ConvertToLegacyMods(attribs.Mods), + attrib_id = (ushort)a.attributeId, + value = Convert.ToSingle(a.value), + }); + } + } + } + + protected void WaitForTotalProcessed(long count, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + if (Processor.TotalProcessed == count) + return; + + Thread.Sleep(50); + } + + throw new XunitException("All scores were not successfully processed"); + } + + protected void WaitForDatabaseState(string sql, T expected, CancellationToken cancellationToken, object? param = null) + { + using (var db = Processor.GetDatabaseConnection()) + { + T? lastValue = default; + + while (true) + { + if (!Debugger.IsAttached) + { + if (cancellationToken.IsCancellationRequested) + throw new TimeoutException($"Waiting for database state took too long (expected: {expected} last: {lastValue} sql: {sql})"); + } + + lastValue = db.QueryFirstOrDefault(sql, param); + + if ((expected == null && lastValue == null) || expected?.Equals(lastValue) == true) + return; + + firstError?.Rethrow(); + + Thread.Sleep(50); + } + } + } + + private void processorOnError(Exception? exception, ScoreItem _) => firstError ??= exception; + +#pragma warning disable CA1816 + public virtual void Dispose() +#pragma warning restore CA1816 + { + cancellationSource.Cancel(); + } + } +} From f514a3c22bfad686a2e2702616ed0383fbbccd3b Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 17:57:14 +0100 Subject: [PATCH 13/29] add test --- .../PerformanceProcessorTests.cs | 1205 +++++++++-------- 1 file changed, 618 insertions(+), 587 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs index c75a996e..e01a7171 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs @@ -1,587 +1,618 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System.Text; -using System.Threading.Tasks; -using Dapper; -using Dapper.Contrib.Extensions; -using MySqlConnector; -using osu.Framework.Extensions.TypeExtensions; -using osu.Framework.Localisation; -using osu.Game.Online.API; -using osu.Game.Rulesets.Catch.Mods; -using osu.Game.Rulesets.Mania.Mods; -using osu.Game.Rulesets.Mods; -using osu.Game.Rulesets.Osu.Difficulty; -using osu.Game.Rulesets.Osu.Mods; -using osu.Game.Rulesets.Scoring; -using osu.Game.Rulesets.Taiko.Difficulty; -using osu.Game.Rulesets.Taiko.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using osu.Server.Queues.ScoreStatisticsProcessor.Processors; -using Xunit; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests -{ - public class PerformanceProcessorTests : DatabaseTest - { - public PerformanceProcessorTests() - { - using (var db = Processor.GetDatabaseConnection()) - { - db.Execute("TRUNCATE TABLE osu_scores_high"); - db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); - } - } - - [Fact] - public void PerformanceIndexUpdates() - { - AddBeatmap(); - AddBeatmapAttributes(); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM osu_user_stats WHERE rank_score > 0 AND user_id = 2", 1, CancellationToken); - WaitForDatabaseState("SELECT rank_score_index FROM osu_user_stats WHERE user_id = 2", 1, CancellationToken); - } - - [Fact] - public void PerformanceDoesNotDoubleAfterScoreSetOnSameMap() - { - AddBeatmap(); - AddBeatmapAttributes(setup: attr => - { - attr.AimDifficulty = 3; - attr.SpeedDifficulty = 3; - attr.OverallDifficulty = 3; - }); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 165pp from the single score above + 2pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 167, CancellationToken); - - // purposefully identical to score above, to confirm that you don't get pp for two scores on the same map twice - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 165pp from the single score above + 4pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 169, CancellationToken); - } - - /// - /// Dear reader: - /// This test will likely appear very random to you. However it in fact exercises a very particular code path. - /// Both client- and server-side components do some rather gnarly juggling to convert between the various score-shaped models - /// (`SoloScore`, `ScoreInfo`, `SoloScoreInfo`...) - /// For most difficulty calculators this doesn't matter because they access fairly simple properties. - /// However taiko pp calculation code has _convert detection_ inside. - /// Therefore it is _very_ important that the single particular access path that the taiko pp calculator uses right now - /// (https://github.com/ppy/osu/blob/555305bf7f650a3461df1e23832ff99b94ca710e/osu.Game.Rulesets.Taiko/Difficulty/TaikoPerformanceCalculator.cs#L44-L45) - /// has the ID of the ruleset for the beatmap BEFORE CONVERSION. - /// This attempts to exercise that requirement in a bit of a dodgy way so that nobody silently breaks taiko pp on accident. - /// - [Fact] - public void TestTaikoNonConvertsCalculatePPCorrectly() - { - AddBeatmap(b => b.playmode = 1); - AddBeatmapAttributes(setup: attr => - { - attr.StaminaDifficulty = 3; - attr.RhythmDifficulty = 3; - attr.ColourDifficulty = 3; - attr.PeakDifficulty = 3; - attr.GreatHitWindow = 15; - }, mode: 1); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ruleset_id = 1; - score.Score.ScoreData.Mods = [new APIMod(new TaikoModHidden())]; - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 298pp from the single score above + 2pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats_taiko WHERE user_id = 2", 300, CancellationToken); - } - - [Fact] - public void LegacyModsThatGivePpAreAllowed() - { - var mods = new Mod[] - { - // Osu - new OsuModEasy(), - new OsuModNoFail(), - new OsuModHalfTime(), - new OsuModHardRock(), - new OsuModSuddenDeath(), - new OsuModPerfect(), - new OsuModDoubleTime(), - new OsuModNightcore(), - new OsuModHidden(), - new OsuModFlashlight(), - new OsuModSpunOut(), - // Taiko - new TaikoModEasy(), - new TaikoModNoFail(), - new TaikoModHalfTime(), - new TaikoModHardRock(), - new TaikoModSuddenDeath(), - new TaikoModPerfect(), - new TaikoModDoubleTime(), - new TaikoModNightcore(), - new TaikoModHidden(), - new TaikoModFlashlight(), - // Catch - new CatchModEasy(), - new CatchModNoFail(), - new CatchModHalfTime(), - new CatchModHardRock(), - new CatchModSuddenDeath(), - new CatchModPerfect(), - new CatchModDoubleTime(), - new CatchModNightcore(), - new CatchModHidden(), - new CatchModFlashlight(), - // Mania - new ManiaModEasy(), - new ManiaModNoFail(), - new ManiaModHalfTime(), - new ManiaModSuddenDeath(), - new ManiaModKey4(), - new ManiaModKey5(), - new ManiaModKey6(), - new ManiaModKey7(), - new ManiaModKey8(), - new ManiaModKey9(), - new ManiaModMirror(), - }; - - foreach (var mod in mods) - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void LegacyModsThatDoNotGivePpAreDisallowed() - { - var mods = new Mod[] - { - // Osu - new OsuModRelax(), - new OsuModAutopilot(), - new OsuModTargetPractice(), - new OsuModAutoplay(), - new OsuModCinema(), - // Taiko - new TaikoModRelax(), - new TaikoModAutoplay(), - // Catch - new CatchModRelax(), - new CatchModAutoplay(), - // Mania - new ManiaModHardRock(), - new ManiaModKey1(), - new ManiaModKey2(), - new ManiaModKey3(), - new ManiaModKey10(), - new ManiaModDualStages(), - new ManiaModRandom(), - new ManiaModAutoplay(), - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ModsThatDoNotGivePpAreDisallowed() - { - // Not an extensive list. - var mods = new Mod[] - { - new ModWindUp(), - new ModWindDown(), - // Osu - new OsuModDeflate(), - new OsuModApproachDifferent(), - new OsuModDifficultyAdjust(), - // Taiko - new TaikoModRandom(), - new TaikoModSwap(), - // Catch - new CatchModMirror(), - new CatchModFloatingFruits(), - new CatchModDifficultyAdjust(), - // Mania - new ManiaModInvert(), - new ManiaModConstantSpeed(), - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ModsThatGivePpAreAllowed() - { - // Not an extensive list. - var mods = new Mod[] - { - // Osu - new OsuModMuted(), - new OsuModDaycore(), - // Taiko - new TaikoModMuted(), - new TaikoModDaycore(), - // Catch - new CatchModMuted(), - new CatchModDaycore(), - // Mania - new ManiaModMuted(), - new ManiaModDaycore(), - }; - - foreach (var mod in mods) - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ClassicAllowedOnLegacyScores() - { - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore { legacy_score_id = 1 }, new Mod[] { new OsuModClassic() })); - } - - [Fact] - public void ClassicDisallowedOnNonLegacyScores() - { - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new Mod[] { new OsuModClassic() })); - } - - [Fact] - public void ModsWithSettingsAreDisallowed() - { - var mods = new Mod[] - { - new OsuModDoubleTime { SpeedChange = { Value = 1.1 } }, - new OsuModClassic { NoSliderHeadAccuracy = { Value = false } }, - new OsuModFlashlight { SizeMultiplier = { Value = 2 } } - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void FailedScoreDoesNotProcess() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.passed = false; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 0, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact(Skip = "ScorePerformanceProcessor is disabled for legacy scores for now: https://github.com/ppy/osu-queue-score-statistics/pull/212#issuecomment-2011297448.")] - public void LegacyScoreIsProcessedAndPpIsWrittenBackToLegacyTables() - { - AddBeatmap(); - AddBeatmapAttributes(); - - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - conn.Execute("INSERT INTO osu_scores_high (score_id, user_id) VALUES (1, 0)"); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.legacy_score_id = 1; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL AND ranked = 1 AND preserve = 1", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM osu_scores_high WHERE score_id = 1 AND pp IS NOT NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact] - public void NonLegacyScoreWithNoBuildIdIsNotRanked() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact] - public void ScoresThatHavePpButInvalidModsGetsNoPP() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score; - - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - { - score = CreateTestScore(beatmapId: TEST_BEATMAP_ID); - - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.ScoreData.Mods = new[] { new APIMod(new InvalidMod()) }; - score.Score.preserve = true; - - conn.Insert(score.Score); - - PushToQueueAndWaitForProcess(score); - } - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Theory] - [InlineData(null, 799)] - [InlineData(0, 799)] - [InlineData(850, 799)] - [InlineData(150, 150)] - public async Task UserHighestRankUpdates(int? highestRankBefore, int highestRankAfter) - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - - if (highestRankBefore != null) - { - await db.ExecuteAsync("INSERT INTO `osu_user_performance_rank_highest` (`user_id`, `mode`, `rank`) VALUES (@userId, @mode, @rank)", new - { - userId = 2, - mode = 0, - rank = highestRankBefore.Value, - }); - } - - var beatmap = AddBeatmap(); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 200; // ~202 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank` FROM `osu_user_performance_rank_highest` WHERE `user_id` = @userId AND `mode` = @mode", highestRankAfter, CancellationToken, new - { - userId = 2, - mode = 0, - }); - } - - [Fact] - public async Task RankIndexPartitionCaching() - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - // Each fake user is spaced 25 pp apart. - // This knowledge can be used to deduce expected values of following assertions. - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {25 * (1000 - i)}, {i + 1}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - - var firstBeatmap = AddBeatmap(b => b.beatmap_id = 1); - - SetScoreForBeatmap(firstBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 150; // ~152 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 995, CancellationToken, new - { - userId = 2, - }); - - SetScoreForBeatmap(firstBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 180; // ~184 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 994, CancellationToken, new - { - userId = 2, - }); - - var secondBeatmap = AddBeatmap(b => b.beatmap_id = 2); - - SetScoreForBeatmap(secondBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 300; // ~486 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 982, CancellationToken, new - { - userId = 2, - }); - } - - [Fact] - public async Task UserDailyRankUpdates() - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 13)"); - - var beatmap = AddBeatmap(); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 200; // ~202 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 799, CancellationToken, new - { - userId = 2, - mode = 0, - }); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 400; // ~404 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 597, CancellationToken, new - { - userId = 2, - mode = 0, - }); - - await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 14)"); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 600; // ~606 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13`, `r14` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", (597, 395), CancellationToken, new - { - userId = 2, - mode = 0, - }); - } - - private class InvalidMod : Mod - { - public override string Name => "Invalid"; - public override LocalisableString Description => "Invalid"; - public override double ScoreMultiplier => 1; - public override string Acronym => "INVALID"; - } - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Text; +using System.Threading.Tasks; +using Dapper; +using Dapper.Contrib.Extensions; +using MySqlConnector; +using osu.Framework.Extensions.TypeExtensions; +using osu.Framework.Localisation; +using osu.Game.Online.API; +using osu.Game.Rulesets.Catch.Mods; +using osu.Game.Rulesets.Mania.Mods; +using osu.Game.Rulesets.Mods; +using osu.Game.Rulesets.Osu.Difficulty; +using osu.Game.Rulesets.Osu.Mods; +using osu.Game.Rulesets.Scoring; +using osu.Game.Rulesets.Taiko.Difficulty; +using osu.Game.Rulesets.Taiko.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using osu.Server.Queues.ScoreStatisticsProcessor.Processors; +using Xunit; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests +{ + public class PerformanceProcessorTests : DatabaseTest + { + public PerformanceProcessorTests() + { + using (var db = Processor.GetDatabaseConnection()) + { + db.Execute("TRUNCATE TABLE osu_scores_high"); + db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); + } + } + + [Fact] + public void PerformanceIndexUpdates() + { + AddBeatmap(); + AddBeatmapAttributes(); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM osu_user_stats WHERE rank_score > 0 AND user_id = 2", 1, CancellationToken); + WaitForDatabaseState("SELECT rank_score_index FROM osu_user_stats WHERE user_id = 2", 1, CancellationToken); + } + + [Fact] + public void PerformanceDoesNotDoubleAfterScoreSetOnSameMap() + { + AddBeatmap(); + AddBeatmapAttributes(setup: attr => + { + attr.AimDifficulty = 3; + attr.SpeedDifficulty = 3; + attr.OverallDifficulty = 3; + }); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 165pp from the single score above + 2pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 167, CancellationToken); + + // purposefully identical to score above, to confirm that you don't get pp for two scores on the same map twice + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 165pp from the single score above + 4pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 169, CancellationToken); + } + + /// + /// Dear reader: + /// This test will likely appear very random to you. However it in fact exercises a very particular code path. + /// Both client- and server-side components do some rather gnarly juggling to convert between the various score-shaped models + /// (`SoloScore`, `ScoreInfo`, `SoloScoreInfo`...) + /// For most difficulty calculators this doesn't matter because they access fairly simple properties. + /// However taiko pp calculation code has _convert detection_ inside. + /// Therefore it is _very_ important that the single particular access path that the taiko pp calculator uses right now + /// (https://github.com/ppy/osu/blob/555305bf7f650a3461df1e23832ff99b94ca710e/osu.Game.Rulesets.Taiko/Difficulty/TaikoPerformanceCalculator.cs#L44-L45) + /// has the ID of the ruleset for the beatmap BEFORE CONVERSION. + /// This attempts to exercise that requirement in a bit of a dodgy way so that nobody silently breaks taiko pp on accident. + /// + [Fact] + public void TestTaikoNonConvertsCalculatePPCorrectly() + { + AddBeatmap(b => b.playmode = 1); + AddBeatmapAttributes(setup: attr => + { + attr.StaminaDifficulty = 3; + attr.RhythmDifficulty = 3; + attr.ColourDifficulty = 3; + attr.PeakDifficulty = 3; + attr.GreatHitWindow = 15; + }, mode: 1); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ruleset_id = 1; + score.Score.ScoreData.Mods = [new APIMod(new TaikoModHidden())]; + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 298pp from the single score above + 2pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats_taiko WHERE user_id = 2", 300, CancellationToken); + } + + [Fact] + public void LegacyModsThatGivePpAreAllowed() + { + var mods = new Mod[] + { + // Osu + new OsuModEasy(), + new OsuModNoFail(), + new OsuModHalfTime(), + new OsuModHardRock(), + new OsuModSuddenDeath(), + new OsuModPerfect(), + new OsuModDoubleTime(), + new OsuModNightcore(), + new OsuModHidden(), + new OsuModFlashlight(), + new OsuModSpunOut(), + // Taiko + new TaikoModEasy(), + new TaikoModNoFail(), + new TaikoModHalfTime(), + new TaikoModHardRock(), + new TaikoModSuddenDeath(), + new TaikoModPerfect(), + new TaikoModDoubleTime(), + new TaikoModNightcore(), + new TaikoModHidden(), + new TaikoModFlashlight(), + // Catch + new CatchModEasy(), + new CatchModNoFail(), + new CatchModHalfTime(), + new CatchModHardRock(), + new CatchModSuddenDeath(), + new CatchModPerfect(), + new CatchModDoubleTime(), + new CatchModNightcore(), + new CatchModHidden(), + new CatchModFlashlight(), + // Mania + new ManiaModEasy(), + new ManiaModNoFail(), + new ManiaModHalfTime(), + new ManiaModSuddenDeath(), + new ManiaModKey4(), + new ManiaModKey5(), + new ManiaModKey6(), + new ManiaModKey7(), + new ManiaModKey8(), + new ManiaModKey9(), + new ManiaModMirror(), + }; + + foreach (var mod in mods) + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void LegacyModsThatDoNotGivePpAreDisallowed() + { + var mods = new Mod[] + { + // Osu + new OsuModRelax(), + new OsuModAutopilot(), + new OsuModTargetPractice(), + new OsuModAutoplay(), + new OsuModCinema(), + // Taiko + new TaikoModRelax(), + new TaikoModAutoplay(), + // Catch + new CatchModRelax(), + new CatchModAutoplay(), + // Mania + new ManiaModHardRock(), + new ManiaModKey1(), + new ManiaModKey2(), + new ManiaModKey3(), + new ManiaModKey10(), + new ManiaModDualStages(), + new ManiaModRandom(), + new ManiaModAutoplay(), + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ModsThatDoNotGivePpAreDisallowed() + { + // Not an extensive list. + var mods = new Mod[] + { + new ModWindUp(), + new ModWindDown(), + // Osu + new OsuModDeflate(), + new OsuModApproachDifferent(), + new OsuModDifficultyAdjust(), + // Taiko + new TaikoModRandom(), + new TaikoModSwap(), + // Catch + new CatchModMirror(), + new CatchModFloatingFruits(), + new CatchModDifficultyAdjust(), + // Mania + new ManiaModInvert(), + new ManiaModConstantSpeed(), + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ModsThatGivePpAreAllowed() + { + // Not an extensive list. + var mods = new Mod[] + { + // Osu + new OsuModMuted(), + new OsuModDaycore(), + // Taiko + new TaikoModMuted(), + new TaikoModDaycore(), + // Catch + new CatchModMuted(), + new CatchModDaycore(), + // Mania + new ManiaModMuted(), + new ManiaModDaycore(), + }; + + foreach (var mod in mods) + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ClassicAllowedOnLegacyScores() + { + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore { legacy_score_id = 1 }, new Mod[] { new OsuModClassic() })); + } + + [Fact] + public void ClassicDisallowedOnNonLegacyScores() + { + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new Mod[] { new OsuModClassic() })); + } + + [Fact] + public void ModsWithSettingsAreDisallowed() + { + var mods = new Mod[] + { + new OsuModDoubleTime { SpeedChange = { Value = 1.1 } }, + new OsuModClassic { NoSliderHeadAccuracy = { Value = false } }, + new OsuModFlashlight { SizeMultiplier = { Value = 2 } } + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void FailedScoreDoesNotProcess() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.passed = false; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 0, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact(Skip = "ScorePerformanceProcessor is disabled for legacy scores for now: https://github.com/ppy/osu-queue-score-statistics/pull/212#issuecomment-2011297448.")] + public void LegacyScoreIsProcessedAndPpIsWrittenBackToLegacyTables() + { + AddBeatmap(); + AddBeatmapAttributes(); + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + conn.Execute("INSERT INTO osu_scores_high (score_id, user_id) VALUES (1, 0)"); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.legacy_score_id = 1; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL AND ranked = 1 AND preserve = 1", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM osu_scores_high WHERE score_id = 1 AND pp IS NOT NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact] + public void NonLegacyScoreWithNoBuildIdIsNotRanked() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact] + public void ScoresThatHavePpButInvalidModsGetsNoPP() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score; + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + score = CreateTestScore(beatmapId: TEST_BEATMAP_ID); + + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.ScoreData.Mods = new[] { new APIMod(new InvalidMod()) }; + score.Score.preserve = true; + + conn.Insert(score.Score); + + PushToQueueAndWaitForProcess(score); + } + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Theory] + [InlineData(null, 799)] + [InlineData(0, 799)] + [InlineData(850, 799)] + [InlineData(150, 150)] + public async Task UserHighestRankUpdates(int? highestRankBefore, int highestRankAfter) + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + + if (highestRankBefore != null) + { + await db.ExecuteAsync("INSERT INTO `osu_user_performance_rank_highest` (`user_id`, `mode`, `rank`) VALUES (@userId, @mode, @rank)", new + { + userId = 2, + mode = 0, + rank = highestRankBefore.Value, + }); + } + + var beatmap = AddBeatmap(); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 200; // ~202 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank` FROM `osu_user_performance_rank_highest` WHERE `user_id` = @userId AND `mode` = @mode", highestRankAfter, CancellationToken, new + { + userId = 2, + mode = 0, + }); + } + + [Fact] + public async Task RankIndexPartitionCaching() + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + // Each fake user is spaced 25 pp apart. + // This knowledge can be used to deduce expected values of following assertions. + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {25 * (1000 - i)}, {i + 1}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + + var firstBeatmap = AddBeatmap(b => b.beatmap_id = 1); + + SetScoreForBeatmap(firstBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 150; // ~152 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 995, CancellationToken, new + { + userId = 2, + }); + + SetScoreForBeatmap(firstBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 180; // ~184 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 994, CancellationToken, new + { + userId = 2, + }); + + var secondBeatmap = AddBeatmap(b => b.beatmap_id = 2); + + SetScoreForBeatmap(secondBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 300; // ~486 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 982, CancellationToken, new + { + userId = 2, + }); + } + + [Fact] + public async Task UserDailyRankUpdates() + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 13)"); + + var beatmap = AddBeatmap(); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 200; // ~202 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 799, CancellationToken, new + { + userId = 2, + mode = 0, + }); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 400; // ~404 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 597, CancellationToken, new + { + userId = 2, + mode = 0, + }); + + await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 14)"); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 600; // ~606 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13`, `r14` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", (597, 395), CancellationToken, new + { + userId = 2, + mode = 0, + }); + } + + [Fact] + public void EnforcedRealtimeDifficultyModsAwardPP() + { + AddBeatmap(b => b.beatmap_id = 315); + + // difficulty attributes are intentionally not added to the database + // so if pp was populated, it had to go through the realtime calculations + + ScoreItem score; + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + score = CreateTestScore(rulesetId: 0, beatmapId: 315); + + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.ScoreData.Mods = new[] { new APIMod(new OsuModMuted()), new APIMod(new OsuModTraceable()) }; + score.Score.preserve = true; + + conn.Insert(score.Score); + PushToQueueAndWaitForProcess(score); + } + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + private class InvalidMod : Mod + { + public override string Name => "Invalid"; + public override LocalisableString Description => "Invalid"; + public override double ScoreMultiplier => 1; + public override string Acronym => "INVALID"; + } + } +} From 26636d0f5e87f282bc1753faafaf0770def51eeb Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 17:57:33 +0100 Subject: [PATCH 14/29] line endings --- .../PerformanceProcessorTests.cs | 1236 ++++++++--------- 1 file changed, 618 insertions(+), 618 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs index e01a7171..29190406 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs @@ -1,618 +1,618 @@ -// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. -// See the LICENCE file in the repository root for full licence text. - -using System.Text; -using System.Threading.Tasks; -using Dapper; -using Dapper.Contrib.Extensions; -using MySqlConnector; -using osu.Framework.Extensions.TypeExtensions; -using osu.Framework.Localisation; -using osu.Game.Online.API; -using osu.Game.Rulesets.Catch.Mods; -using osu.Game.Rulesets.Mania.Mods; -using osu.Game.Rulesets.Mods; -using osu.Game.Rulesets.Osu.Difficulty; -using osu.Game.Rulesets.Osu.Mods; -using osu.Game.Rulesets.Scoring; -using osu.Game.Rulesets.Taiko.Difficulty; -using osu.Game.Rulesets.Taiko.Mods; -using osu.Server.Queues.ScoreStatisticsProcessor.Models; -using osu.Server.Queues.ScoreStatisticsProcessor.Processors; -using Xunit; - -namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests -{ - public class PerformanceProcessorTests : DatabaseTest - { - public PerformanceProcessorTests() - { - using (var db = Processor.GetDatabaseConnection()) - { - db.Execute("TRUNCATE TABLE osu_scores_high"); - db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); - } - } - - [Fact] - public void PerformanceIndexUpdates() - { - AddBeatmap(); - AddBeatmapAttributes(); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM osu_user_stats WHERE rank_score > 0 AND user_id = 2", 1, CancellationToken); - WaitForDatabaseState("SELECT rank_score_index FROM osu_user_stats WHERE user_id = 2", 1, CancellationToken); - } - - [Fact] - public void PerformanceDoesNotDoubleAfterScoreSetOnSameMap() - { - AddBeatmap(); - AddBeatmapAttributes(setup: attr => - { - attr.AimDifficulty = 3; - attr.SpeedDifficulty = 3; - attr.OverallDifficulty = 3; - }); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 165pp from the single score above + 2pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 167, CancellationToken); - - // purposefully identical to score above, to confirm that you don't get pp for two scores on the same map twice - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 165pp from the single score above + 4pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 169, CancellationToken); - } - - /// - /// Dear reader: - /// This test will likely appear very random to you. However it in fact exercises a very particular code path. - /// Both client- and server-side components do some rather gnarly juggling to convert between the various score-shaped models - /// (`SoloScore`, `ScoreInfo`, `SoloScoreInfo`...) - /// For most difficulty calculators this doesn't matter because they access fairly simple properties. - /// However taiko pp calculation code has _convert detection_ inside. - /// Therefore it is _very_ important that the single particular access path that the taiko pp calculator uses right now - /// (https://github.com/ppy/osu/blob/555305bf7f650a3461df1e23832ff99b94ca710e/osu.Game.Rulesets.Taiko/Difficulty/TaikoPerformanceCalculator.cs#L44-L45) - /// has the ID of the ruleset for the beatmap BEFORE CONVERSION. - /// This attempts to exercise that requirement in a bit of a dodgy way so that nobody silently breaks taiko pp on accident. - /// - [Fact] - public void TestTaikoNonConvertsCalculatePPCorrectly() - { - AddBeatmap(b => b.playmode = 1); - AddBeatmapAttributes(setup: attr => - { - attr.StaminaDifficulty = 3; - attr.RhythmDifficulty = 3; - attr.ColourDifficulty = 3; - attr.PeakDifficulty = 3; - attr.GreatHitWindow = 15; - }, mode: 1); - - SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ruleset_id = 1; - score.Score.ScoreData.Mods = [new APIMod(new TaikoModHidden())]; - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.preserve = true; - }); - - // 298pp from the single score above + 2pp from playcount bonus - WaitForDatabaseState("SELECT rank_score FROM osu_user_stats_taiko WHERE user_id = 2", 300, CancellationToken); - } - - [Fact] - public void LegacyModsThatGivePpAreAllowed() - { - var mods = new Mod[] - { - // Osu - new OsuModEasy(), - new OsuModNoFail(), - new OsuModHalfTime(), - new OsuModHardRock(), - new OsuModSuddenDeath(), - new OsuModPerfect(), - new OsuModDoubleTime(), - new OsuModNightcore(), - new OsuModHidden(), - new OsuModFlashlight(), - new OsuModSpunOut(), - // Taiko - new TaikoModEasy(), - new TaikoModNoFail(), - new TaikoModHalfTime(), - new TaikoModHardRock(), - new TaikoModSuddenDeath(), - new TaikoModPerfect(), - new TaikoModDoubleTime(), - new TaikoModNightcore(), - new TaikoModHidden(), - new TaikoModFlashlight(), - // Catch - new CatchModEasy(), - new CatchModNoFail(), - new CatchModHalfTime(), - new CatchModHardRock(), - new CatchModSuddenDeath(), - new CatchModPerfect(), - new CatchModDoubleTime(), - new CatchModNightcore(), - new CatchModHidden(), - new CatchModFlashlight(), - // Mania - new ManiaModEasy(), - new ManiaModNoFail(), - new ManiaModHalfTime(), - new ManiaModSuddenDeath(), - new ManiaModKey4(), - new ManiaModKey5(), - new ManiaModKey6(), - new ManiaModKey7(), - new ManiaModKey8(), - new ManiaModKey9(), - new ManiaModMirror(), - }; - - foreach (var mod in mods) - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void LegacyModsThatDoNotGivePpAreDisallowed() - { - var mods = new Mod[] - { - // Osu - new OsuModRelax(), - new OsuModAutopilot(), - new OsuModTargetPractice(), - new OsuModAutoplay(), - new OsuModCinema(), - // Taiko - new TaikoModRelax(), - new TaikoModAutoplay(), - // Catch - new CatchModRelax(), - new CatchModAutoplay(), - // Mania - new ManiaModHardRock(), - new ManiaModKey1(), - new ManiaModKey2(), - new ManiaModKey3(), - new ManiaModKey10(), - new ManiaModDualStages(), - new ManiaModRandom(), - new ManiaModAutoplay(), - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ModsThatDoNotGivePpAreDisallowed() - { - // Not an extensive list. - var mods = new Mod[] - { - new ModWindUp(), - new ModWindDown(), - // Osu - new OsuModDeflate(), - new OsuModApproachDifferent(), - new OsuModDifficultyAdjust(), - // Taiko - new TaikoModRandom(), - new TaikoModSwap(), - // Catch - new CatchModMirror(), - new CatchModFloatingFruits(), - new CatchModDifficultyAdjust(), - // Mania - new ManiaModInvert(), - new ManiaModConstantSpeed(), - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ModsThatGivePpAreAllowed() - { - // Not an extensive list. - var mods = new Mod[] - { - // Osu - new OsuModMuted(), - new OsuModDaycore(), - // Taiko - new TaikoModMuted(), - new TaikoModDaycore(), - // Catch - new CatchModMuted(), - new CatchModDaycore(), - // Mania - new ManiaModMuted(), - new ManiaModDaycore(), - }; - - foreach (var mod in mods) - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void ClassicAllowedOnLegacyScores() - { - Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore { legacy_score_id = 1 }, new Mod[] { new OsuModClassic() })); - } - - [Fact] - public void ClassicDisallowedOnNonLegacyScores() - { - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new Mod[] { new OsuModClassic() })); - } - - [Fact] - public void ModsWithSettingsAreDisallowed() - { - var mods = new Mod[] - { - new OsuModDoubleTime { SpeedChange = { Value = 1.1 } }, - new OsuModClassic { NoSliderHeadAccuracy = { Value = false } }, - new OsuModFlashlight { SizeMultiplier = { Value = 2 } } - }; - - foreach (var mod in mods) - Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); - } - - [Fact] - public void FailedScoreDoesNotProcess() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.passed = false; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 0, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact(Skip = "ScorePerformanceProcessor is disabled for legacy scores for now: https://github.com/ppy/osu-queue-score-statistics/pull/212#issuecomment-2011297448.")] - public void LegacyScoreIsProcessedAndPpIsWrittenBackToLegacyTables() - { - AddBeatmap(); - AddBeatmapAttributes(); - - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - conn.Execute("INSERT INTO osu_scores_high (score_id, user_id) VALUES (1, 0)"); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.legacy_score_id = 1; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL AND ranked = 1 AND preserve = 1", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM osu_scores_high WHERE score_id = 1 AND pp IS NOT NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact] - public void NonLegacyScoreWithNoBuildIdIsNotRanked() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => - { - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.preserve = true; - }); - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Fact] - public void ScoresThatHavePpButInvalidModsGetsNoPP() - { - AddBeatmap(); - AddBeatmapAttributes(); - - ScoreItem score; - - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - { - score = CreateTestScore(beatmapId: TEST_BEATMAP_ID); - - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.ScoreData.Mods = new[] { new APIMod(new InvalidMod()) }; - score.Score.preserve = true; - - conn.Insert(score.Score); - - PushToQueueAndWaitForProcess(score); - } - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - [Theory] - [InlineData(null, 799)] - [InlineData(0, 799)] - [InlineData(850, 799)] - [InlineData(150, 150)] - public async Task UserHighestRankUpdates(int? highestRankBefore, int highestRankAfter) - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - - if (highestRankBefore != null) - { - await db.ExecuteAsync("INSERT INTO `osu_user_performance_rank_highest` (`user_id`, `mode`, `rank`) VALUES (@userId, @mode, @rank)", new - { - userId = 2, - mode = 0, - rank = highestRankBefore.Value, - }); - } - - var beatmap = AddBeatmap(); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 200; // ~202 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank` FROM `osu_user_performance_rank_highest` WHERE `user_id` = @userId AND `mode` = @mode", highestRankAfter, CancellationToken, new - { - userId = 2, - mode = 0, - }); - } - - [Fact] - public async Task RankIndexPartitionCaching() - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - // Each fake user is spaced 25 pp apart. - // This knowledge can be used to deduce expected values of following assertions. - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {25 * (1000 - i)}, {i + 1}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - - var firstBeatmap = AddBeatmap(b => b.beatmap_id = 1); - - SetScoreForBeatmap(firstBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 150; // ~152 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 995, CancellationToken, new - { - userId = 2, - }); - - SetScoreForBeatmap(firstBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 180; // ~184 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 994, CancellationToken, new - { - userId = 2, - }); - - var secondBeatmap = AddBeatmap(b => b.beatmap_id = 2); - - SetScoreForBeatmap(secondBeatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 300; // ~486 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 982, CancellationToken, new - { - userId = 2, - }); - } - - [Fact] - public async Task UserDailyRankUpdates() - { - // simulate fake users to beat as we climb up ranks. - // this is going to be a bit of a chonker query... - using var db = Processor.GetDatabaseConnection(); - - var stringBuilder = new StringBuilder(); - - stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " - + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " - + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); - - for (int i = 0; i < 1000; ++i) - { - if (i > 0) - stringBuilder.Append(','); - - stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); - } - - await db.ExecuteAsync(stringBuilder.ToString()); - await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 13)"); - - var beatmap = AddBeatmap(); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 200; // ~202 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 799, CancellationToken, new - { - userId = 2, - mode = 0, - }); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 400; // ~404 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 597, CancellationToken, new - { - userId = 2, - mode = 0, - }); - - await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 14)"); - - SetScoreForBeatmap(beatmap.beatmap_id, s => - { - s.Score.preserve = s.Score.ranked = true; - s.Score.pp = 600; // ~606 pp total, including bonus pp - }); - - WaitForDatabaseState("SELECT `r13`, `r14` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", (597, 395), CancellationToken, new - { - userId = 2, - mode = 0, - }); - } - - [Fact] - public void EnforcedRealtimeDifficultyModsAwardPP() - { - AddBeatmap(b => b.beatmap_id = 315); - - // difficulty attributes are intentionally not added to the database - // so if pp was populated, it had to go through the realtime calculations - - ScoreItem score; - - using (MySqlConnection conn = Processor.GetDatabaseConnection()) - { - score = CreateTestScore(rulesetId: 0, beatmapId: 315); - - score.Score.ScoreData.Statistics[HitResult.Great] = 100; - score.Score.max_combo = 100; - score.Score.accuracy = 1; - score.Score.build_id = TestBuildID; - score.Score.ScoreData.Mods = new[] { new APIMod(new OsuModMuted()), new APIMod(new OsuModTraceable()) }; - score.Score.preserve = true; - - conn.Insert(score.Score); - PushToQueueAndWaitForProcess(score); - } - - WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 1, CancellationToken, new - { - ScoreId = score.Score.id - }); - } - - private class InvalidMod : Mod - { - public override string Name => "Invalid"; - public override LocalisableString Description => "Invalid"; - public override double ScoreMultiplier => 1; - public override string Acronym => "INVALID"; - } - } -} +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Text; +using System.Threading.Tasks; +using Dapper; +using Dapper.Contrib.Extensions; +using MySqlConnector; +using osu.Framework.Extensions.TypeExtensions; +using osu.Framework.Localisation; +using osu.Game.Online.API; +using osu.Game.Rulesets.Catch.Mods; +using osu.Game.Rulesets.Mania.Mods; +using osu.Game.Rulesets.Mods; +using osu.Game.Rulesets.Osu.Difficulty; +using osu.Game.Rulesets.Osu.Mods; +using osu.Game.Rulesets.Scoring; +using osu.Game.Rulesets.Taiko.Difficulty; +using osu.Game.Rulesets.Taiko.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using osu.Server.Queues.ScoreStatisticsProcessor.Processors; +using Xunit; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests +{ + public class PerformanceProcessorTests : DatabaseTest + { + public PerformanceProcessorTests() + { + using (var db = Processor.GetDatabaseConnection()) + { + db.Execute("TRUNCATE TABLE osu_scores_high"); + db.Execute("TRUNCATE TABLE osu_beatmap_difficulty_attribs"); + } + } + + [Fact] + public void PerformanceIndexUpdates() + { + AddBeatmap(); + AddBeatmapAttributes(); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM osu_user_stats WHERE rank_score > 0 AND user_id = 2", 1, CancellationToken); + WaitForDatabaseState("SELECT rank_score_index FROM osu_user_stats WHERE user_id = 2", 1, CancellationToken); + } + + [Fact] + public void PerformanceDoesNotDoubleAfterScoreSetOnSameMap() + { + AddBeatmap(); + AddBeatmapAttributes(setup: attr => + { + attr.AimDifficulty = 3; + attr.SpeedDifficulty = 3; + attr.OverallDifficulty = 3; + }); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 165pp from the single score above + 2pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 167, CancellationToken); + + // purposefully identical to score above, to confirm that you don't get pp for two scores on the same map twice + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 165pp from the single score above + 4pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats WHERE user_id = 2", 169, CancellationToken); + } + + /// + /// Dear reader: + /// This test will likely appear very random to you. However it in fact exercises a very particular code path. + /// Both client- and server-side components do some rather gnarly juggling to convert between the various score-shaped models + /// (`SoloScore`, `ScoreInfo`, `SoloScoreInfo`...) + /// For most difficulty calculators this doesn't matter because they access fairly simple properties. + /// However taiko pp calculation code has _convert detection_ inside. + /// Therefore it is _very_ important that the single particular access path that the taiko pp calculator uses right now + /// (https://github.com/ppy/osu/blob/555305bf7f650a3461df1e23832ff99b94ca710e/osu.Game.Rulesets.Taiko/Difficulty/TaikoPerformanceCalculator.cs#L44-L45) + /// has the ID of the ruleset for the beatmap BEFORE CONVERSION. + /// This attempts to exercise that requirement in a bit of a dodgy way so that nobody silently breaks taiko pp on accident. + /// + [Fact] + public void TestTaikoNonConvertsCalculatePPCorrectly() + { + AddBeatmap(b => b.playmode = 1); + AddBeatmapAttributes(setup: attr => + { + attr.StaminaDifficulty = 3; + attr.RhythmDifficulty = 3; + attr.ColourDifficulty = 3; + attr.PeakDifficulty = 3; + attr.GreatHitWindow = 15; + }, mode: 1); + + SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ruleset_id = 1; + score.Score.ScoreData.Mods = [new APIMod(new TaikoModHidden())]; + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.preserve = true; + }); + + // 298pp from the single score above + 2pp from playcount bonus + WaitForDatabaseState("SELECT rank_score FROM osu_user_stats_taiko WHERE user_id = 2", 300, CancellationToken); + } + + [Fact] + public void LegacyModsThatGivePpAreAllowed() + { + var mods = new Mod[] + { + // Osu + new OsuModEasy(), + new OsuModNoFail(), + new OsuModHalfTime(), + new OsuModHardRock(), + new OsuModSuddenDeath(), + new OsuModPerfect(), + new OsuModDoubleTime(), + new OsuModNightcore(), + new OsuModHidden(), + new OsuModFlashlight(), + new OsuModSpunOut(), + // Taiko + new TaikoModEasy(), + new TaikoModNoFail(), + new TaikoModHalfTime(), + new TaikoModHardRock(), + new TaikoModSuddenDeath(), + new TaikoModPerfect(), + new TaikoModDoubleTime(), + new TaikoModNightcore(), + new TaikoModHidden(), + new TaikoModFlashlight(), + // Catch + new CatchModEasy(), + new CatchModNoFail(), + new CatchModHalfTime(), + new CatchModHardRock(), + new CatchModSuddenDeath(), + new CatchModPerfect(), + new CatchModDoubleTime(), + new CatchModNightcore(), + new CatchModHidden(), + new CatchModFlashlight(), + // Mania + new ManiaModEasy(), + new ManiaModNoFail(), + new ManiaModHalfTime(), + new ManiaModSuddenDeath(), + new ManiaModKey4(), + new ManiaModKey5(), + new ManiaModKey6(), + new ManiaModKey7(), + new ManiaModKey8(), + new ManiaModKey9(), + new ManiaModMirror(), + }; + + foreach (var mod in mods) + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void LegacyModsThatDoNotGivePpAreDisallowed() + { + var mods = new Mod[] + { + // Osu + new OsuModRelax(), + new OsuModAutopilot(), + new OsuModTargetPractice(), + new OsuModAutoplay(), + new OsuModCinema(), + // Taiko + new TaikoModRelax(), + new TaikoModAutoplay(), + // Catch + new CatchModRelax(), + new CatchModAutoplay(), + // Mania + new ManiaModHardRock(), + new ManiaModKey1(), + new ManiaModKey2(), + new ManiaModKey3(), + new ManiaModKey10(), + new ManiaModDualStages(), + new ManiaModRandom(), + new ManiaModAutoplay(), + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ModsThatDoNotGivePpAreDisallowed() + { + // Not an extensive list. + var mods = new Mod[] + { + new ModWindUp(), + new ModWindDown(), + // Osu + new OsuModDeflate(), + new OsuModApproachDifferent(), + new OsuModDifficultyAdjust(), + // Taiko + new TaikoModRandom(), + new TaikoModSwap(), + // Catch + new CatchModMirror(), + new CatchModFloatingFruits(), + new CatchModDifficultyAdjust(), + // Mania + new ManiaModInvert(), + new ManiaModConstantSpeed(), + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ModsThatGivePpAreAllowed() + { + // Not an extensive list. + var mods = new Mod[] + { + // Osu + new OsuModMuted(), + new OsuModDaycore(), + // Taiko + new TaikoModMuted(), + new TaikoModDaycore(), + // Catch + new CatchModMuted(), + new CatchModDaycore(), + // Mania + new ManiaModMuted(), + new ManiaModDaycore(), + }; + + foreach (var mod in mods) + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void ClassicAllowedOnLegacyScores() + { + Assert.True(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore { legacy_score_id = 1 }, new Mod[] { new OsuModClassic() })); + } + + [Fact] + public void ClassicDisallowedOnNonLegacyScores() + { + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new Mod[] { new OsuModClassic() })); + } + + [Fact] + public void ModsWithSettingsAreDisallowed() + { + var mods = new Mod[] + { + new OsuModDoubleTime { SpeedChange = { Value = 1.1 } }, + new OsuModClassic { NoSliderHeadAccuracy = { Value = false } }, + new OsuModFlashlight { SizeMultiplier = { Value = 2 } } + }; + + foreach (var mod in mods) + Assert.False(ScorePerformanceProcessor.AllModsValidForPerformance(new SoloScore(), new[] { mod }), mod.GetType().ReadableName()); + } + + [Fact] + public void FailedScoreDoesNotProcess() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.passed = false; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 0, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact(Skip = "ScorePerformanceProcessor is disabled for legacy scores for now: https://github.com/ppy/osu-queue-score-statistics/pull/212#issuecomment-2011297448.")] + public void LegacyScoreIsProcessedAndPpIsWrittenBackToLegacyTables() + { + AddBeatmap(); + AddBeatmapAttributes(); + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + conn.Execute("INSERT INTO osu_scores_high (score_id, user_id) VALUES (1, 0)"); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.legacy_score_id = 1; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL AND ranked = 1 AND preserve = 1", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM osu_scores_high WHERE score_id = 1 AND pp IS NOT NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact] + public void NonLegacyScoreWithNoBuildIdIsNotRanked() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score = SetScoreForBeatmap(TEST_BEATMAP_ID, score => + { + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.preserve = true; + }); + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Fact] + public void ScoresThatHavePpButInvalidModsGetsNoPP() + { + AddBeatmap(); + AddBeatmapAttributes(); + + ScoreItem score; + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + score = CreateTestScore(beatmapId: TEST_BEATMAP_ID); + + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.ScoreData.Mods = new[] { new APIMod(new InvalidMod()) }; + score.Score.preserve = true; + + conn.Insert(score.Score); + + PushToQueueAndWaitForProcess(score); + } + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + [Theory] + [InlineData(null, 799)] + [InlineData(0, 799)] + [InlineData(850, 799)] + [InlineData(150, 150)] + public async Task UserHighestRankUpdates(int? highestRankBefore, int highestRankAfter) + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + + if (highestRankBefore != null) + { + await db.ExecuteAsync("INSERT INTO `osu_user_performance_rank_highest` (`user_id`, `mode`, `rank`) VALUES (@userId, @mode, @rank)", new + { + userId = 2, + mode = 0, + rank = highestRankBefore.Value, + }); + } + + var beatmap = AddBeatmap(); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 200; // ~202 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank` FROM `osu_user_performance_rank_highest` WHERE `user_id` = @userId AND `mode` = @mode", highestRankAfter, CancellationToken, new + { + userId = 2, + mode = 0, + }); + } + + [Fact] + public async Task RankIndexPartitionCaching() + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + // Each fake user is spaced 25 pp apart. + // This knowledge can be used to deduce expected values of following assertions. + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {25 * (1000 - i)}, {i + 1}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + + var firstBeatmap = AddBeatmap(b => b.beatmap_id = 1); + + SetScoreForBeatmap(firstBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 150; // ~152 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 995, CancellationToken, new + { + userId = 2, + }); + + SetScoreForBeatmap(firstBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 180; // ~184 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 994, CancellationToken, new + { + userId = 2, + }); + + var secondBeatmap = AddBeatmap(b => b.beatmap_id = 2); + + SetScoreForBeatmap(secondBeatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 300; // ~486 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `rank_score_index` FROM `osu_user_stats` WHERE `user_id` = @userId", 982, CancellationToken, new + { + userId = 2, + }); + } + + [Fact] + public async Task UserDailyRankUpdates() + { + // simulate fake users to beat as we climb up ranks. + // this is going to be a bit of a chonker query... + using var db = Processor.GetDatabaseConnection(); + + var stringBuilder = new StringBuilder(); + + stringBuilder.Append("INSERT INTO osu_user_stats (`user_id`, `rank_score`, `rank_score_index`, " + + "`accuracy_total`, `accuracy_count`, `accuracy`, `accuracy_new`, `playcount`, `ranked_score`, `total_score`, " + + "`x_rank_count`, `xh_rank_count`, `s_rank_count`, `sh_rank_count`, `a_rank_count`, `rank`, `level`) VALUES "); + + for (int i = 0; i < 1000; ++i) + { + if (i > 0) + stringBuilder.Append(','); + + stringBuilder.Append($"({1000 + i}, {1000 - i}, {i}, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1)"); + } + + await db.ExecuteAsync(stringBuilder.ToString()); + await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 13)"); + + var beatmap = AddBeatmap(); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 200; // ~202 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 799, CancellationToken, new + { + userId = 2, + mode = 0, + }); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 400; // ~404 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", 597, CancellationToken, new + { + userId = 2, + mode = 0, + }); + + await db.ExecuteAsync("REPLACE INTO `osu_counts` (name, count) VALUES ('pp_rank_column_osu', 14)"); + + SetScoreForBeatmap(beatmap.beatmap_id, s => + { + s.Score.preserve = s.Score.ranked = true; + s.Score.pp = 600; // ~606 pp total, including bonus pp + }); + + WaitForDatabaseState("SELECT `r13`, `r14` FROM `osu_user_performance_rank` WHERE `user_id` = @userId AND `mode` = @mode", (597, 395), CancellationToken, new + { + userId = 2, + mode = 0, + }); + } + + [Fact] + public void EnforcedRealtimeDifficultyModsAwardPP() + { + AddBeatmap(b => b.beatmap_id = 315); + + // difficulty attributes are intentionally not added to the database + // so if pp was populated, it had to go through the realtime calculations + + ScoreItem score; + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + score = CreateTestScore(rulesetId: 0, beatmapId: 315); + + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.ScoreData.Mods = new[] { new APIMod(new OsuModMuted()), new APIMod(new OsuModTraceable()) }; + score.Score.preserve = true; + + conn.Insert(score.Score); + PushToQueueAndWaitForProcess(score); + } + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + + private class InvalidMod : Mod + { + public override string Name => "Invalid"; + public override LocalisableString Description => "Invalid"; + public override double ScoreMultiplier => 1; + public override string Acronym => "INVALID"; + } + } +} From 0ec8c9692a43aaeb18b5fec77063a42b1217da01 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:40:42 +0100 Subject: [PATCH 15/29] enforce realtime difficulty calculation when using non-default configuration on mods add additional support for realtime calculations on non-legacy mods line endings --- README.md | 4 +- .../Stores/BeatmapStore.cs | 49 +++++++++++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b28435e5..a55b43a1 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ Whether to process user total stats. Set to `0` to disable processing. Default is unset (processing enabled). -### REALTIME_DIFFICULTY +### ALWAYS_REALTIME_DIFFICULTY -Whether to use realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data. Set to `0` to disable processing. +Whether to use **always** realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. Default is unset (processing enabled). diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 59d503f8..68506ede 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading.Tasks; @@ -17,6 +18,7 @@ using osu.Game.Rulesets.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; +using StatsdClient; using Beatmap = osu.Server.Queues.ScoreStatisticsProcessor.Models.Beatmap; namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores @@ -26,7 +28,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores /// public class BeatmapStore { - private static readonly bool use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("REALTIME_DIFFICULTY") != "0"; + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); @@ -65,8 +67,16 @@ public static async Task CreateAsync(MySqlConnection connection, M /// The difficulty attributes or null if not existing. public async Task GetDifficultyAttributesAsync(Beatmap beatmap, Ruleset ruleset, Mod[] mods, MySqlConnection connection, MySqlTransaction? transaction = null) { - if (use_realtime_difficulty_calculation) + // database attributes are stored using the default mod configurations + // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) + // or non-legacy mods which aren't populated into the database + // then we must calculate difficulty attributes in real-time. + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { + var stopwatch = Stopwatch.StartNew(); + using var req = new WebRequest(string.Format(beatmap_download_path, beatmap.beatmap_id)); req.AllowInsecureRequests = true; @@ -79,7 +89,17 @@ public static async Task CreateAsync(MySqlConnection connection, M var workingBeatmap = new StreamedWorkingBeatmap(req.ResponseStream); var calculator = ruleset.CreateDifficultyCalculator(workingBeatmap); - return calculator.Calculate(mods); + var attributes = calculator.Calculate(mods); + + string[] tags = + { + $"ruleset:{ruleset.RulesetInfo.OnlineID}", + $"mods:{string.Join("", mods.Select(x => x.Acronym))}" + }; + + DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + + return attributes; } BeatmapDifficultyAttribute[]? rawDifficultyAttributes; @@ -107,12 +127,33 @@ public static async Task CreateAsync(MySqlConnection connection, M return difficultyAttributes; } + /// + /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// Used by to decide if the current mod combination's difficulty attributes + /// can be fetched from the database. + /// + private static bool isLegacyMod(Mod mod) => + mod is ModNoFail + or ModEasy + or ModHidden + or ModHardRock + or ModPerfect + or ModSuddenDeath + or ModNightcore + or ModDoubleTime + or ModRelax + or ModHalfTime + or ModFlashlight + or ModCinema + or ModAutoplay + or ModScoreV2; + /// /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. /// The match is not always exact; for some mods that award pp but do not exist in stable /// (such as ) the closest available approximation is used. /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . + /// The entirety of this workaround is not used / unnecessary if is . /// private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) { From 12217e7246f1163d756ff946efcf76b6dd5623b9 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 18:54:37 +0100 Subject: [PATCH 16/29] remove unneeded interpolation line endings --- .../Stores/BeatmapStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 68506ede..73c6d49d 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -97,7 +97,7 @@ public static async Task CreateAsync(MySqlConnection connection, M $"mods:{string.Join("", mods.Select(x => x.Acronym))}" }; - DogStatsd.Timer($"calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); + DogStatsd.Timer("calculate-realtime-difficulty-attributes", stopwatch.ElapsedMilliseconds, tags: tags); return attributes; } From be5a3349e60174eb55a073570cd7f168761f9145 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 15 Jul 2024 19:49:59 +0100 Subject: [PATCH 17/29] fix CL edge case --- .../Stores/BeatmapStore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 73c6d49d..1186ecc3 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -69,9 +69,9 @@ public static async Task CreateAsync(MySqlConnection connection, M { // database attributes are stored using the default mod configurations // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) - // or non-legacy mods which aren't populated into the database + // or non-legacy mods which aren't populated into the database (with exception to CL) // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || !isLegacyMod(m)); + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isLegacyMod(m) && m is not ModClassic)); if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { From c03acf17110f690e1833e27623ac00d931a6ff83 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:22:53 +0100 Subject: [PATCH 18/29] rename `isLegacyMod` to `isRankedLegacyMod` and add missing mania mods line endings (sigh) --- .../Stores/BeatmapStore.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 1186ecc3..dfafebce 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -15,6 +15,7 @@ using osu.Game.Beatmaps.Legacy; using osu.Game.Rulesets; using osu.Game.Rulesets.Difficulty; +using osu.Game.Rulesets.Mania.Mods; using osu.Game.Rulesets.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; @@ -71,7 +72,7 @@ public static async Task CreateAsync(MySqlConnection connection, M // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) // or non-legacy mods which aren't populated into the database (with exception to CL) // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isLegacyMod(m) && m is not ModClassic)); + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { @@ -128,25 +129,29 @@ public static async Task CreateAsync(MySqlConnection connection, M } /// - /// This method attempts to create a simple solution to deciding if a can be considered a "legacy" mod. + /// This method attempts to create a simple solution to deciding if a can be considered a ranked "legacy" mod. /// Used by to decide if the current mod combination's difficulty attributes /// can be fetched from the database. /// - private static bool isLegacyMod(Mod mod) => + private static bool isRankedLegacyMod(Mod mod) => mod is ModNoFail or ModEasy - or ModHidden + or ModHidden // this also catches ManiaModFadeIn or ModHardRock or ModPerfect or ModSuddenDeath or ModNightcore or ModDoubleTime - or ModRelax or ModHalfTime or ModFlashlight - or ModCinema - or ModAutoplay - or ModScoreV2; + or ModTouchDevice + or ManiaModKey4 + or ManiaModKey5 + or ManiaModKey6 + or ManiaModKey7 + or ManiaModKey8 + or ManiaModKey9 + or ManiaModMirror; /// /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. From ff2aa5958731c5bcfa6a85d660c5a98cae1a7590 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:28:02 +0100 Subject: [PATCH 19/29] add `OsuModSpunOut` --- .../Stores/BeatmapStore.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index dfafebce..538924ba 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -17,6 +17,7 @@ using osu.Game.Rulesets.Difficulty; using osu.Game.Rulesets.Mania.Mods; using osu.Game.Rulesets.Mods; +using osu.Game.Rulesets.Osu.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; using StatsdClient; @@ -145,6 +146,7 @@ or ModDoubleTime or ModHalfTime or ModFlashlight or ModTouchDevice + or OsuModSpunOut or ManiaModKey4 or ManiaModKey5 or ManiaModKey6 From 7bbb452f98e8e736ec0409d651718824bac49616 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 10:32:30 +0100 Subject: [PATCH 20/29] handle hard rock being unranked for mania --- .../Stores/BeatmapStore.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 538924ba..6499d3d4 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -14,10 +14,12 @@ using osu.Game.Beatmaps; using osu.Game.Beatmaps.Legacy; using osu.Game.Rulesets; +using osu.Game.Rulesets.Catch.Mods; using osu.Game.Rulesets.Difficulty; using osu.Game.Rulesets.Mania.Mods; using osu.Game.Rulesets.Mods; using osu.Game.Rulesets.Osu.Mods; +using osu.Game.Rulesets.Taiko.Mods; using osu.Server.Queues.ScoreStatisticsProcessor.Helpers; using osu.Server.Queues.ScoreStatisticsProcessor.Models; using StatsdClient; @@ -138,7 +140,6 @@ private static bool isRankedLegacyMod(Mod mod) => mod is ModNoFail or ModEasy or ModHidden // this also catches ManiaModFadeIn - or ModHardRock or ModPerfect or ModSuddenDeath or ModNightcore @@ -146,7 +147,10 @@ or ModDoubleTime or ModHalfTime or ModFlashlight or ModTouchDevice + or OsuModHardRock or OsuModSpunOut + or TaikoModHardRock + or CatchModHardRock or ManiaModKey4 or ManiaModKey5 or ManiaModKey6 From b3ed26087cb8c1d59943261b984ee58c84bb6a1d Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 11:44:48 +0100 Subject: [PATCH 21/29] use `ALWAYS_REALTIME_DIFFICULTY` in `DatabaseTest` line endings --- .../DatabaseTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index f597910f..35a4a819 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -45,7 +45,7 @@ protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) ? new CancellationTokenSource() : new CancellationTokenSource(20000); - Environment.SetEnvironmentVariable("REALTIME_DIFFICULTY", "0"); + Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); Processor.Error += processorOnError; From fc5e32ae24db5073c3bc8018555fa773fc4e1c89 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Tue, 16 Jul 2024 17:57:14 +0100 Subject: [PATCH 22/29] add test line endings --- .../PerformanceProcessorTests.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs index c75a996e..29190406 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/PerformanceProcessorTests.cs @@ -576,6 +576,37 @@ public async Task UserDailyRankUpdates() }); } + [Fact] + public void EnforcedRealtimeDifficultyModsAwardPP() + { + AddBeatmap(b => b.beatmap_id = 315); + + // difficulty attributes are intentionally not added to the database + // so if pp was populated, it had to go through the realtime calculations + + ScoreItem score; + + using (MySqlConnection conn = Processor.GetDatabaseConnection()) + { + score = CreateTestScore(rulesetId: 0, beatmapId: 315); + + score.Score.ScoreData.Statistics[HitResult.Great] = 100; + score.Score.max_combo = 100; + score.Score.accuracy = 1; + score.Score.build_id = TestBuildID; + score.Score.ScoreData.Mods = new[] { new APIMod(new OsuModMuted()), new APIMod(new OsuModTraceable()) }; + score.Score.preserve = true; + + conn.Insert(score.Score); + PushToQueueAndWaitForProcess(score); + } + + WaitForDatabaseState("SELECT COUNT(*) FROM scores WHERE id = @ScoreId AND pp IS NOT NULL", 1, CancellationToken, new + { + ScoreId = score.Score.id + }); + } + private class InvalidMod : Mod { public override string Name => "Invalid"; From 2f562cd5197cc5fe300a90b9fd7cacf7b490e7f8 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Wed, 17 Jul 2024 08:13:40 +0100 Subject: [PATCH 23/29] rename env var to `PREFER_REALTIME_DIFFICULTY` --- README.md | 4 ++-- .../DatabaseTest.cs | 2 +- .../Stores/BeatmapStore.cs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a55b43a1..406017bc 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ Whether to process user total stats. Set to `0` to disable processing. Default is unset (processing enabled). -### ALWAYS_REALTIME_DIFFICULTY +### PREFER_REALTIME_DIFFICULTY -Whether to use **always** realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. +Whether to use prefer realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. Default is unset (processing enabled). diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index 35a4a819..c0dabe9a 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -45,7 +45,7 @@ protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) ? new CancellationTokenSource() : new CancellationTokenSource(20000); - Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); + Environment.SetEnvironmentVariable("PREFER_REALTIME_DIFFICULTY", "0"); Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); Processor.Error += processorOnError; diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 6499d3d4..2cb4c1b3 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -32,7 +32,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores /// public class BeatmapStore { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly bool prefer_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("PREFER_REALTIME_DIFFICULTY") != "0"; private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); @@ -77,7 +77,7 @@ public static async Task CreateAsync(MySqlConnection connection, M // then we must calculate difficulty attributes in real-time. bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); - if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + if (prefer_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { var stopwatch = Stopwatch.StartNew(); @@ -164,7 +164,7 @@ or ManiaModKey9 /// The match is not always exact; for some mods that award pp but do not exist in stable /// (such as ) the closest available approximation is used. /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . + /// The entirety of this workaround is not used / unnecessary if is . /// private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) { From 808dc5fcd927ecda0450e0e457cf78e660510898 Mon Sep 17 00:00:00 2001 From: James Wilson Date: Wed, 17 Jul 2024 12:32:45 +0100 Subject: [PATCH 24/29] fix README grammar --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 406017bc..1e9378ca 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Default is unset (processing enabled). ### PREFER_REALTIME_DIFFICULTY -Whether to use prefer realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. +Whether to prefer realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. Default is unset (processing enabled). From 19f72cb5e86fba4f3d9778bb1b450d9c64de99d9 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 22 Jul 2024 09:25:18 +0100 Subject: [PATCH 25/29] move prefer naming back to always --- README.md | 4 ++-- .../DatabaseTest.cs | 2 +- .../Stores/BeatmapStore.cs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1e9378ca..75871df5 100644 --- a/README.md +++ b/README.md @@ -119,9 +119,9 @@ Whether to process user total stats. Set to `0` to disable processing. Default is unset (processing enabled). -### PREFER_REALTIME_DIFFICULTY +### ALWAYS_REALTIME_DIFFICULTY -Whether to prefer realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. +Whether to always use realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. Default is unset (processing enabled). diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index c0dabe9a..35a4a819 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -45,7 +45,7 @@ protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) ? new CancellationTokenSource() : new CancellationTokenSource(20000); - Environment.SetEnvironmentVariable("PREFER_REALTIME_DIFFICULTY", "0"); + Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); Processor.Error += processorOnError; diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 2cb4c1b3..6499d3d4 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -32,7 +32,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores /// public class BeatmapStore { - private static readonly bool prefer_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("PREFER_REALTIME_DIFFICULTY") != "0"; + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); @@ -77,7 +77,7 @@ public static async Task CreateAsync(MySqlConnection connection, M // then we must calculate difficulty attributes in real-time. bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); - if (prefer_realtime_difficulty_calculation || mustUseRealtimeDifficulty) + if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { var stopwatch = Stopwatch.StartNew(); @@ -164,7 +164,7 @@ or ManiaModKey9 /// The match is not always exact; for some mods that award pp but do not exist in stable /// (such as ) the closest available approximation is used. /// Moreover, the set of returned is constrained to mods that actually affect difficulty in the legacy sense. - /// The entirety of this workaround is not used / unnecessary if is . + /// The entirety of this workaround is not used / unnecessary if is . /// private static LegacyMods getLegacyModsForAttributeLookup(Beatmap beatmap, Ruleset ruleset, Mod[] mods) { From 2abc9b2da20029527dacaf027c405e3b7ec223d3 Mon Sep 17 00:00:00 2001 From: tsunyoku Date: Mon, 22 Jul 2024 12:16:49 +0100 Subject: [PATCH 26/29] rename to ALWAYS_USE_REALTIME_DIFFICULTY --- README.md | 2 +- .../DatabaseTest.cs | 2 +- .../Stores/BeatmapStore.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 75871df5..68554a4e 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Whether to process user total stats. Set to `0` to disable processing. Default is unset (processing enabled). -### ALWAYS_REALTIME_DIFFICULTY +### ALWAYS_USE_REALTIME_DIFFICULTY Whether to always use realtime processing (download beatmaps and compute their difficulty attributes on every processed score), or to rely on database data when possible. Set to `0` to disable processing. diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index 35a4a819..a80d139a 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -45,7 +45,7 @@ protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) ? new CancellationTokenSource() : new CancellationTokenSource(20000); - Environment.SetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY", "0"); + Environment.SetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY ", "0"); Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); Processor.Error += processorOnError; diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 6499d3d4..95ff6777 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -32,7 +32,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores /// public class BeatmapStore { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_REALTIME_DIFFICULTY") != "0"; + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY ") != "0"; private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary(); From 09efc446d830650e0d28142b1c394058c64cfade Mon Sep 17 00:00:00 2001 From: James Wilson Date: Thu, 1 Aug 2024 20:00:01 +0200 Subject: [PATCH 27/29] more precise legacy mod checking --- .../Stores/BeatmapStore.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 95ff6777..02e38eb6 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -139,7 +139,6 @@ public static async Task CreateAsync(MySqlConnection connection, M private static bool isRankedLegacyMod(Mod mod) => mod is ModNoFail or ModEasy - or ModHidden // this also catches ManiaModFadeIn or ModPerfect or ModSuddenDeath or ModNightcore @@ -149,15 +148,20 @@ or ModFlashlight or ModTouchDevice or OsuModHardRock or OsuModSpunOut + or OsuModHidden or TaikoModHardRock + or TaikoModHidden or CatchModHardRock + or CatchModHidden or ManiaModKey4 or ManiaModKey5 or ManiaModKey6 or ManiaModKey7 or ManiaModKey8 or ManiaModKey9 - or ManiaModMirror; + or ManiaModMirror + or ManiaModHidden + or ManiaModFadeIn; /// /// This method attempts to choose the best possible set of to use for looking up stored difficulty attributes. From 327669a1ec444100ade5444020d0fa9e44aac7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 7 Aug 2024 10:06:40 +0200 Subject: [PATCH 28/29] Add preventative test coverage guarding against accidental inheritance --- .../BeatmapStoreTest.cs | 105 ++++++++++++++++++ .../Stores/BeatmapStore.cs | 4 +- 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 osu.Server.Queues.ScoreStatisticsProcessor.Tests/BeatmapStoreTest.cs diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/BeatmapStoreTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/BeatmapStoreTest.cs new file mode 100644 index 00000000..f0dad2ca --- /dev/null +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/BeatmapStoreTest.cs @@ -0,0 +1,105 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Collections.Generic; +using osu.Game.Rulesets; +using osu.Game.Rulesets.Catch; +using osu.Game.Rulesets.Catch.Mods; +using osu.Game.Rulesets.Mania; +using osu.Game.Rulesets.Mania.Mods; +using osu.Game.Rulesets.Osu; +using osu.Game.Rulesets.Osu.Mods; +using osu.Game.Rulesets.Taiko; +using osu.Game.Rulesets.Taiko.Mods; +using osu.Server.Queues.ScoreStatisticsProcessor.Stores; +using Xunit; + +namespace osu.Server.Queues.ScoreStatisticsProcessor.Tests +{ + public class BeatmapStoreTest + { + private static readonly HashSet osu_ranked_mods = new HashSet + { + typeof(OsuModNoFail), + typeof(OsuModEasy), + typeof(OsuModTouchDevice), + typeof(OsuModHidden), + typeof(OsuModHardRock), + typeof(OsuModSuddenDeath), + typeof(OsuModDoubleTime), + typeof(OsuModHalfTime), + typeof(OsuModNightcore), + typeof(OsuModFlashlight), + typeof(OsuModSpunOut), + typeof(OsuModPerfect), + }; + + private static readonly HashSet taiko_ranked_mods = new HashSet + { + typeof(TaikoModNoFail), + typeof(TaikoModEasy), + typeof(TaikoModHidden), + typeof(TaikoModHardRock), + typeof(TaikoModSuddenDeath), + typeof(TaikoModDoubleTime), + typeof(TaikoModHalfTime), + typeof(TaikoModNightcore), + typeof(TaikoModFlashlight), + typeof(TaikoModPerfect), + }; + + private static readonly HashSet catch_ranked_mods = new HashSet + { + typeof(CatchModNoFail), + typeof(CatchModEasy), + typeof(CatchModHidden), + typeof(CatchModHardRock), + typeof(CatchModSuddenDeath), + typeof(CatchModDoubleTime), + typeof(CatchModHalfTime), + typeof(CatchModNightcore), + typeof(CatchModFlashlight), + typeof(CatchModPerfect), + }; + + private static readonly HashSet mania_ranked_mods = new HashSet + { + typeof(ManiaModNoFail), + typeof(ManiaModEasy), + typeof(ManiaModHidden), + typeof(ManiaModSuddenDeath), + typeof(ManiaModDoubleTime), + typeof(ManiaModHalfTime), + typeof(ManiaModNightcore), + typeof(ManiaModFlashlight), + typeof(ManiaModPerfect), + typeof(ManiaModKey4), + typeof(ManiaModKey5), + typeof(ManiaModKey6), + typeof(ManiaModKey7), + typeof(ManiaModKey8), + typeof(ManiaModFadeIn), + typeof(ManiaModKey9), + typeof(ManiaModMirror), + }; + + public static readonly object[][] RANKED_TEST_DATA = + [ + [new OsuRuleset(), osu_ranked_mods], + [new TaikoRuleset(), taiko_ranked_mods], + [new CatchRuleset(), catch_ranked_mods], + [new ManiaRuleset(), mania_ranked_mods], + ]; + + [Theory] + [MemberData(nameof(RANKED_TEST_DATA))] + public void TestLegacyModsMarkedAsRankedCorrectly(Ruleset ruleset, HashSet legacyModTypes) + { + var rulesetMods = ruleset.CreateAllMods(); + + foreach (var mod in rulesetMods) + Assert.Equal(legacyModTypes.Contains(mod.GetType()), BeatmapStore.IsRankedLegacyMod(mod)); + } + } +} diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 02e38eb6..53c33b29 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -75,7 +75,7 @@ public static async Task CreateAsync(MySqlConnection connection, M // if we want to support mods with non-default configurations (i.e non-1.5x rates on DT/NC) // or non-legacy mods which aren't populated into the database (with exception to CL) // then we must calculate difficulty attributes in real-time. - bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!isRankedLegacyMod(m) && m is not ModClassic)); + bool mustUseRealtimeDifficulty = mods.Any(m => !m.UsesDefaultConfiguration || (!IsRankedLegacyMod(m) && m is not ModClassic)); if (always_use_realtime_difficulty_calculation || mustUseRealtimeDifficulty) { @@ -136,7 +136,7 @@ public static async Task CreateAsync(MySqlConnection connection, M /// Used by to decide if the current mod combination's difficulty attributes /// can be fetched from the database. /// - private static bool isRankedLegacyMod(Mod mod) => + public static bool IsRankedLegacyMod(Mod mod) => mod is ModNoFail or ModEasy or ModPerfect From 8b57913d86206f2ce686b8fb990550316201a9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bart=C5=82omiej=20Dach?= Date: Wed, 7 Aug 2024 10:42:52 +0200 Subject: [PATCH 29/29] Remove space in envvar name --- .../DatabaseTest.cs | 2 +- .../Stores/BeatmapStore.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs index a80d139a..483e72c8 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor.Tests/DatabaseTest.cs @@ -45,7 +45,7 @@ protected DatabaseTest(AssemblyName[]? externalProcessorAssemblies = null) ? new CancellationTokenSource() : new CancellationTokenSource(20000); - Environment.SetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY ", "0"); + Environment.SetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY", "0"); Processor = new ScoreStatisticsQueueProcessor(externalProcessorAssemblies: externalProcessorAssemblies); Processor.Error += processorOnError; diff --git a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs index 53c33b29..3eb46872 100644 --- a/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs +++ b/osu.Server.Queues.ScoreStatisticsProcessor/Stores/BeatmapStore.cs @@ -32,7 +32,7 @@ namespace osu.Server.Queues.ScoreStatisticsProcessor.Stores /// public class BeatmapStore { - private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY ") != "0"; + private static readonly bool always_use_realtime_difficulty_calculation = Environment.GetEnvironmentVariable("ALWAYS_USE_REALTIME_DIFFICULTY") != "0"; private static readonly string beatmap_download_path = Environment.GetEnvironmentVariable("BEATMAP_DOWNLOAD_PATH") ?? "https://osu.ppy.sh/osu/{0}"; private readonly ConcurrentDictionary beatmapCache = new ConcurrentDictionary();