From b0b39bbd0b4ca6e0423e9d5c04a1b87d552780da Mon Sep 17 00:00:00 2001 From: Noi Date: Sat, 27 May 2023 18:07:45 -0700 Subject: [PATCH] Allow configuration of certain time-based values --- BackgroundServices/AutoUserDownload.cs | 10 +++++----- BackgroundServices/BackgroundService.cs | 12 ++++++++---- BackgroundServices/BirthdayRoleUpdate.cs | 10 +++++----- BackgroundServices/DataRetention.cs | 16 +++++++++------- .../ExternalStatisticsReporting.cs | 17 ++++++++++------- BackgroundServices/ShardBackgroundWorker.cs | 3 ++- Configuration.cs | 18 ++++++++++++++++++ ShardManager.cs | 15 ++------------- 8 files changed, 59 insertions(+), 42 deletions(-) diff --git a/BackgroundServices/AutoUserDownload.cs b/BackgroundServices/AutoUserDownload.cs index c4cb47c..cc7f588 100644 --- a/BackgroundServices/AutoUserDownload.cs +++ b/BackgroundServices/AutoUserDownload.cs @@ -13,14 +13,14 @@ class AutoUserDownload : BackgroundService { public override async Task OnTick(int tickCount, CancellationToken token) { // Take action if a guild's cache is incomplete... - var incompleteCaches = ShardInstance.DiscordClient.Guilds + var incompleteCaches = Shard.DiscordClient.Guilds .Where(g => !g.HasAllMembers) .Select(g => g.Id) .ToHashSet(); // ...and if the guild contains any user data IEnumerable mustFetch; try { - await DbConcurrentOperationsLock.WaitAsync(token); + await ConcurrentSemaphore.WaitAsync(token); using var db = new BotDatabaseContext(); lock (_failedDownloads) mustFetch = db.UserEntries.AsNoTracking() @@ -31,7 +31,7 @@ class AutoUserDownload : BackgroundService { .ToList(); } finally { try { - DbConcurrentOperationsLock.Release(); + ConcurrentSemaphore.Release(); } catch (ObjectDisposedException) { } } @@ -39,9 +39,9 @@ class AutoUserDownload : BackgroundService { var processStartTime = DateTimeOffset.UtcNow; foreach (var item in mustFetch) { // May cause a disconnect in certain situations. Make no further attempts until the next pass if it happens. - if (ShardInstance.DiscordClient.ConnectionState != ConnectionState.Connected) break; + if (Shard.DiscordClient.ConnectionState != ConnectionState.Connected) break; - var guild = ShardInstance.DiscordClient.GetGuild(item); + var guild = Shard.DiscordClient.GetGuild(item); if (guild == null) continue; // A guild disappeared...? await Task.Delay(200, CancellationToken.None); // Delay a bit (reduces the possibility of hanging, somehow). diff --git a/BackgroundServices/BackgroundService.cs b/BackgroundServices/BackgroundService.cs index d2dcce8..bb798bd 100644 --- a/BackgroundServices/BackgroundService.cs +++ b/BackgroundServices/BackgroundService.cs @@ -1,11 +1,15 @@ namespace BirthdayBot.BackgroundServices; abstract class BackgroundService { - protected static SemaphoreSlim DbConcurrentOperationsLock { get; } = new(ShardManager.MaxConcurrentOperations); - protected ShardInstance ShardInstance { get; } + protected static SemaphoreSlim ConcurrentSemaphore { get; private set; } = null!; - public BackgroundService(ShardInstance instance) => ShardInstance = instance; + protected ShardInstance Shard { get; } - protected void Log(string message) => ShardInstance.Log(GetType().Name, message); + public BackgroundService(ShardInstance instance) { + Shard = instance; + ConcurrentSemaphore ??= new SemaphoreSlim(instance.Config.MaxConcurrentOperations); + } + + protected void Log(string message) => Shard.Log(GetType().Name, message); public abstract Task OnTick(int tickCount, CancellationToken token); } diff --git a/BackgroundServices/BirthdayRoleUpdate.cs b/BackgroundServices/BirthdayRoleUpdate.cs index 25f621b..d3c5bc2 100644 --- a/BackgroundServices/BirthdayRoleUpdate.cs +++ b/BackgroundServices/BirthdayRoleUpdate.cs @@ -15,11 +15,11 @@ class BirthdayRoleUpdate : BackgroundService { /// public override async Task OnTick(int tickCount, CancellationToken token) { try { - await DbConcurrentOperationsLock.WaitAsync(token); + await ConcurrentSemaphore.WaitAsync(token); await ProcessBirthdaysAsync(token); } finally { try { - DbConcurrentOperationsLock.Release(); + ConcurrentSemaphore.Release(); } catch (ObjectDisposedException) { } } } @@ -27,19 +27,19 @@ class BirthdayRoleUpdate : BackgroundService { private async Task ProcessBirthdaysAsync(CancellationToken token) { // For database efficiency, fetch all database information at once before proceeding using var db = new BotDatabaseContext(); - var shardGuilds = ShardInstance.DiscordClient.Guilds.Select(g => g.Id).ToHashSet(); + var shardGuilds = Shard.DiscordClient.Guilds.Select(g => g.Id).ToHashSet(); var presentGuildSettings = db.GuildConfigurations.Where(s => shardGuilds.Contains(s.GuildId)); var guildChecks = presentGuildSettings.ToList().Select(s => Tuple.Create(s.GuildId, s)); var exceptions = new List(); foreach (var (guildId, settings) in guildChecks) { - var guild = ShardInstance.DiscordClient.GetGuild(guildId); + var guild = Shard.DiscordClient.GetGuild(guildId); if (guild == null) continue; // A guild disappeared...? // Check task cancellation here. Processing during a single guild is never interrupted. if (token.IsCancellationRequested) throw new TaskCanceledException(); - if (ShardInstance.DiscordClient.ConnectionState != ConnectionState.Connected) { + if (Shard.DiscordClient.ConnectionState != ConnectionState.Connected) { Log("Client is not connected. Stopping early."); return; } diff --git a/BackgroundServices/DataRetention.cs b/BackgroundServices/DataRetention.cs index 6f0a0e6..1ae8ec4 100644 --- a/BackgroundServices/DataRetention.cs +++ b/BackgroundServices/DataRetention.cs @@ -7,24 +7,26 @@ namespace BirthdayBot.BackgroundServices; /// Automatically removes database information for guilds that have not been accessed in a long time. /// class DataRetention : BackgroundService { - const int ProcessInterval = 5400 / ShardBackgroundWorker.Interval; // Process about once per hour and a half + private readonly int ProcessInterval; // Amount of days without updates before data is considered stale and up for deletion. const int StaleGuildThreshold = 180; const int StaleUserThreashold = 360; - public DataRetention(ShardInstance instance) : base(instance) { } + public DataRetention(ShardInstance instance) : base(instance) { + ProcessInterval = 5400 / Shard.Config.BackgroundInterval; // Process about once per hour and a half + } public override async Task OnTick(int tickCount, CancellationToken token) { // On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks. - if ((tickCount + ShardInstance.ShardId) % ProcessInterval != 0) return; + if ((tickCount + Shard.ShardId) % ProcessInterval != 0) return; try { - await DbConcurrentOperationsLock.WaitAsync(token); + await ConcurrentSemaphore.WaitAsync(token); await RemoveStaleEntriesAsync(); } finally { try { - DbConcurrentOperationsLock.Release(); + ConcurrentSemaphore.Release(); } catch (ObjectDisposedException) { } } } @@ -34,14 +36,14 @@ class DataRetention : BackgroundService { var now = DateTimeOffset.UtcNow; // Update guilds - var localGuilds = ShardInstance.DiscordClient.Guilds.Select(g => g.Id).ToList(); + var localGuilds = Shard.DiscordClient.Guilds.Select(g => g.Id).ToList(); var updatedGuilds = await db.GuildConfigurations .Where(g => localGuilds.Contains(g.GuildId)) .ExecuteUpdateAsync(upd => upd.SetProperty(p => p.LastSeen, now)); // Update guild users var updatedUsers = 0; - foreach (var guild in ShardInstance.DiscordClient.Guilds) { + foreach (var guild in Shard.DiscordClient.Guilds) { var localUsers = guild.Users.Select(u => u.Id).ToList(); updatedUsers += await db.UserEntries .Where(gu => gu.GuildId == guild.Id) diff --git a/BackgroundServices/ExternalStatisticsReporting.cs b/BackgroundServices/ExternalStatisticsReporting.cs index 6374f3d..16145fb 100644 --- a/BackgroundServices/ExternalStatisticsReporting.cs +++ b/BackgroundServices/ExternalStatisticsReporting.cs @@ -5,22 +5,25 @@ namespace BirthdayBot.BackgroundServices; /// Reports user count statistics to external services on a shard by shard basis. /// class ExternalStatisticsReporting : BackgroundService { - const int ProcessInterval = 1200 / ShardBackgroundWorker.Interval; // Process every ~20 minutes - const int ProcessOffset = 300 / ShardBackgroundWorker.Interval; // Begin processing ~5 minutes after shard start + readonly int ProcessInterval; + readonly int ProcessOffset; private static readonly HttpClient _httpClient = new(); - public ExternalStatisticsReporting(ShardInstance instance) : base(instance) { } + public ExternalStatisticsReporting(ShardInstance instance) : base(instance) { + ProcessInterval = 1200 / Shard.Config.BackgroundInterval; // Process every ~20 minutes + ProcessOffset = 300 / Shard.Config.BackgroundInterval; // No processing until ~5 minutes after shard start + } public override async Task OnTick(int tickCount, CancellationToken token) { if (tickCount < ProcessOffset) return; if (tickCount % ProcessInterval != 0) return; - var botId = ShardInstance.DiscordClient.CurrentUser.Id; + var botId = Shard.DiscordClient.CurrentUser.Id; if (botId == 0) return; - var count = ShardInstance.DiscordClient.Guilds.Count; + var count = Shard.DiscordClient.Guilds.Count; - var dbotsToken = ShardInstance.Config.DBotsToken; + var dbotsToken = Shard.Config.DBotsToken; if (dbotsToken != null) await SendDiscordBots(dbotsToken, count, botId, token); } @@ -33,7 +36,7 @@ class ExternalStatisticsReporting : BackgroundService { var post = new HttpRequestMessage(HttpMethod.Post, uri); post.Headers.Add("Authorization", apiToken); post.Content = new StringContent(string.Format(Body, - userCount, ShardInstance.Config.ShardTotal, ShardInstance.ShardId), + userCount, Shard.Config.ShardTotal, Shard.ShardId), Encoding.UTF8, "application/json"); await _httpClient.SendAsync(post, token); diff --git a/BackgroundServices/ShardBackgroundWorker.cs b/BackgroundServices/ShardBackgroundWorker.cs index f9a1860..2c6f2f9 100644 --- a/BackgroundServices/ShardBackgroundWorker.cs +++ b/BackgroundServices/ShardBackgroundWorker.cs @@ -6,7 +6,7 @@ class ShardBackgroundWorker : IDisposable { /// /// The interval, in seconds, in which background tasks are attempted to be run within a shard. /// - public const int Interval = 40; + private int Interval { get; } private readonly Task _workerTask; private readonly CancellationTokenSource _workerCanceller; @@ -20,6 +20,7 @@ class ShardBackgroundWorker : IDisposable { public ShardBackgroundWorker(ShardInstance instance) { Instance = instance; + Interval = instance.Config.BackgroundInterval; _workerCanceller = new CancellationTokenSource(); _workers = new List() diff --git a/Configuration.cs b/Configuration.cs index 94623a5..5683708 100644 --- a/Configuration.cs +++ b/Configuration.cs @@ -25,6 +25,20 @@ class Configuration { public string SqlPassword { get; } internal string SqlApplicationName { get; } + /// + /// Number of seconds between each time the status task runs, in seconds. + /// + public int StatusInterval { get; } + /// + /// Number of concurrent shard startups to happen on each check. + /// This value also determines the maximum amount of concurrent background database operations. + /// + public int MaxConcurrentOperations { get; } + /// + /// Amount of time to wait between background task runs within each shard. + /// + public int BackgroundInterval { get; } + public Configuration() { var args = CommandLineParameters.Parse(Environment.GetCommandLineArgs()); var path = args?.ConfigFile ?? Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location) @@ -71,6 +85,10 @@ class Configuration { SqlUsername = ReadConfKey(jc, nameof(SqlUsername), true); SqlPassword = ReadConfKey(jc, nameof(SqlPassword), true); SqlApplicationName = $"Shard{ShardStart:00}-{ShardStart + ShardAmount - 1:00}"; + + StatusInterval = ReadConfKey(jc, nameof(StatusInterval), false) ?? 90; + MaxConcurrentOperations = ReadConfKey(jc, nameof(MaxConcurrentOperations), false) ?? 4; + BackgroundInterval = ReadConfKey(jc, nameof(BackgroundInterval), false) ?? 60; } private static T? ReadConfKey(JObject jc, string key, [DoesNotReturnIf(true)] bool failOnEmpty) { diff --git a/ShardManager.cs b/ShardManager.cs index 2732204..d742725 100644 --- a/ShardManager.cs +++ b/ShardManager.cs @@ -10,17 +10,6 @@ namespace BirthdayBot; /// status reports regarding the overall health of the application. /// class ShardManager : IDisposable { - /// - /// Number of seconds between each time the status task runs, in seconds. - /// - private const int StatusInterval = 90; - - /// - /// Number of concurrent shard startups to happen on each check. - /// This value also determines the maximum amount of concurrent background database operations. - /// - public const int MaxConcurrentOperations = 4; - /// /// Amount of time without a completed background service run before a shard instance /// is considered "dead" and tasked to be removed. @@ -158,14 +147,14 @@ class ShardManager : IDisposable { } // Start null shards, a few at at time - var startAllowance = MaxConcurrentOperations; + var startAllowance = Config.MaxConcurrentOperations; foreach (var id in nullShards) { if (startAllowance-- > 0) { _shards[id] = await InitializeShard(id); } else break; } - await Task.Delay(StatusInterval * 1000, _mainCancel.Token); + await Task.Delay(Config.StatusInterval * 1000, _mainCancel.Token); } } catch (TaskCanceledException) { } }