From 57a870efdaf9a2230244fe5df13e6b1317274d0f Mon Sep 17 00:00:00 2001 From: Noi Date: Wed, 13 Oct 2021 19:06:29 -0700 Subject: [PATCH] Additional diagnostic information --- BackgroundServices/DataRetention.cs | 9 +++++---- BackgroundServices/ShardBackgroundWorker.cs | 9 ++++++--- ShardInstance.cs | 4 ++++ ShardManager.cs | 18 +++++++++++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/BackgroundServices/DataRetention.cs b/BackgroundServices/DataRetention.cs index 08189e0..6807f04 100644 --- a/BackgroundServices/DataRetention.cs +++ b/BackgroundServices/DataRetention.cs @@ -15,17 +15,18 @@ namespace BirthdayBot.BackgroundServices /// class DataRetention : BackgroundService { - private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(2); - const int ProcessInterval = 600 / ShardBackgroundWorker.Interval; // Process every ~10 minutes - private int _tickCount = 0; + private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(ShardManager.MaxConcurrentOperations); + const int ProcessInterval = 3600 / ShardBackgroundWorker.Interval; // Process about once per hour + private int _tickCount = -1; public DataRetention(ShardInstance instance) : base(instance) { } public override async Task OnTick(CancellationToken token) { - if (++_tickCount % ProcessInterval != 0) + if ((++_tickCount + ShardInstance.ShardId * 3) % ProcessInterval != 0) { // Do not process on every tick. + // Stagger processing based on shard ID, to not choke the background processing task. return; } diff --git a/BackgroundServices/ShardBackgroundWorker.cs b/BackgroundServices/ShardBackgroundWorker.cs index 946fd76..f5d4b2f 100644 --- a/BackgroundServices/ShardBackgroundWorker.cs +++ b/BackgroundServices/ShardBackgroundWorker.cs @@ -25,6 +25,7 @@ namespace BirthdayBot.BackgroundServices public BirthdayRoleUpdate BirthdayUpdater { get; } public SelectiveAutoUserDownload UserDownloader { get; } public DateTimeOffset LastBackgroundRun { get; private set; } + public string CurrentExecutingService { get; private set; } public int ConnectionScore => ConnStatus.Score; public ShardBackgroundWorker(ShardInstance instance) @@ -76,6 +77,7 @@ namespace BirthdayBot.BackgroundServices // Execute tasks sequentially foreach (var service in _workers) { + CurrentExecutingService = service.GetType().Name; try { if (_workerCanceller.IsCancellationRequested) break; @@ -83,19 +85,20 @@ namespace BirthdayBot.BackgroundServices } catch (Exception ex) { - var svcname = service.GetType().Name; + if (ex is TaskCanceledException) { - Instance.Log(nameof(WorkerLoop), $"{svcname} was interrupted by a cancellation request."); + Instance.Log(nameof(WorkerLoop), $"{CurrentExecutingService} was interrupted by a cancellation request."); throw; } else { // TODO webhook log - Instance.Log(nameof(WorkerLoop), $"{svcname} encountered an exception:\n" + ex.ToString()); + Instance.Log(nameof(WorkerLoop), $"{CurrentExecutingService} encountered an exception:\n" + ex.ToString()); } } } + CurrentExecutingService = null; LastBackgroundRun = DateTimeOffset.UtcNow; } } diff --git a/ShardInstance.cs b/ShardInstance.cs index 9482b5d..ecafa1c 100644 --- a/ShardInstance.cs +++ b/ShardInstance.cs @@ -25,6 +25,10 @@ namespace BirthdayBot /// Returns a value showing the time in which the last background run successfully completed. /// public DateTimeOffset LastBackgroundRun => _background.LastBackgroundRun; + /// + /// Returns the name of the background service currently in execution. + /// + public string CurrentExecutingService => _background.CurrentExecutingService; public Configuration Config => _manager.Config; /// /// Returns this shard's connection score. diff --git a/ShardManager.cs b/ShardManager.cs index 3f055a5..f3cc415 100644 --- a/ShardManager.cs +++ b/ShardManager.cs @@ -27,7 +27,13 @@ namespace BirthdayBot /// /// Number of shards allowed to be destroyed before forcing the program to close. /// - private const int MaxDestroyedShards = 5; + private const int MaxDestroyedShards = 10; // TODO make configurable + + /// + /// Number of concurrent shard startups to happen on each check. + /// This value is also used in . + /// + public const int MaxConcurrentOperations = 5; /// /// Amount of time without a completed background service run before a shard instance @@ -145,7 +151,7 @@ namespace BirthdayBot Log($"Bot uptime: {Common.BotUptime}"); // Iterate through shard list, extract data - var guildInfo = new Dictionary(); + var guildInfo = new Dictionary(); var now = DateTimeOffset.UtcNow; var nullShards = new List(); foreach (var item in _shards) @@ -160,8 +166,9 @@ namespace BirthdayBot var guildCount = shard.DiscordClient.Guilds.Count; var connScore = shard.ConnectionScore; var lastRun = now - shard.LastBackgroundRun; + var lastExec = shard.CurrentExecutingService ?? "null"; - guildInfo[item.Key] = (guildCount, connScore, lastRun); + guildInfo[item.Key] = (guildCount, connScore, lastRun, lastExec); } // Process info @@ -203,7 +210,8 @@ namespace BirthdayBot { result.Remove(result.Length - 1, 1); result.Append($"[{guildInfo[item].Item2:+0;-0}"); - result.Append($" {Math.Floor(guildInfo[item].Item3.TotalSeconds):000}s] "); + result.Append($" {Math.Floor(guildInfo[item].Item3.TotalSeconds):000}s"); + result.Append($" {guildInfo[item].Item4}] "); } } if (result.Length > 0) result.Remove(result.Length - 1, 1); @@ -228,7 +236,7 @@ namespace BirthdayBot else { // Start up any missing shards - int startAllowance = 4; + int startAllowance = MaxConcurrentOperations; foreach (var id in nullShards) { // To avoid possible issues with resources strained over so many shards starting at once,