Additional diagnostic information

BackgroundServices/DataRetention.cs

@@ -15,17 +15,18 @@ namespace BirthdayBot.BackgroundServices
     /// </summary>
     class DataRetention : BackgroundService
     {
-        private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(2);
+        private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(ShardManager.MaxConcurrentOperations);
-        const int ProcessInterval = 600 / ShardBackgroundWorker.Interval; // Process every ~10 minutes
+        const int ProcessInterval = 3600 / ShardBackgroundWorker.Interval; // Process about once per hour
-        private int _tickCount = 0;
+        private int _tickCount = -1;
 
         public DataRetention(ShardInstance instance) : base(instance) { }
 
         public override async Task OnTick(CancellationToken token)
         {
-            if (++_tickCount % ProcessInterval != 0)
+            if ((++_tickCount + ShardInstance.ShardId * 3) % ProcessInterval != 0)
             {
                 // Do not process on every tick.
+                // Stagger processing based on shard ID, to not choke the background processing task.
                 return;
             }
 

BackgroundServices/ShardBackgroundWorker.cs

@@ -25,6 +25,7 @@ namespace BirthdayBot.BackgroundServices
         public BirthdayRoleUpdate BirthdayUpdater { get; }
         public SelectiveAutoUserDownload UserDownloader { get; }
         public DateTimeOffset LastBackgroundRun { get; private set; }
+        public string CurrentExecutingService { get; private set; }
         public int ConnectionScore => ConnStatus.Score;
 
         public ShardBackgroundWorker(ShardInstance instance)
@@ -76,6 +77,7 @@ namespace BirthdayBot.BackgroundServices
                     // Execute tasks sequentially
                     foreach (var service in _workers)
                     {
+                        CurrentExecutingService = service.GetType().Name;
                         try
                         {
                             if (_workerCanceller.IsCancellationRequested) break;
@@ -83,19 +85,20 @@ namespace BirthdayBot.BackgroundServices
                         }
                         catch (Exception ex)
                         {
-                            var svcname = service.GetType().Name;
+                            
                             if (ex is TaskCanceledException)
                             {
-                                Instance.Log(nameof(WorkerLoop), $"{svcname} was interrupted by a cancellation request.");
+                                Instance.Log(nameof(WorkerLoop), $"{CurrentExecutingService} was interrupted by a cancellation request.");
                                 throw;
                             }
                             else
                             {
                                 // TODO webhook log
-                                Instance.Log(nameof(WorkerLoop), $"{svcname} encountered an exception:\n" + ex.ToString());
+                                Instance.Log(nameof(WorkerLoop), $"{CurrentExecutingService} encountered an exception:\n" + ex.ToString());
                             }
                         }
                     }
+                    CurrentExecutingService = null;
                     LastBackgroundRun = DateTimeOffset.UtcNow;
                 }
             }

ShardInstance.cs

@@ -25,6 +25,10 @@ namespace BirthdayBot
         /// Returns a value showing the time in which the last background run successfully completed.
         /// </summary>
         public DateTimeOffset LastBackgroundRun => _background.LastBackgroundRun;
+        /// <summary>
+        /// Returns the name of the background service currently in execution.
+        /// </summary>
+        public string CurrentExecutingService => _background.CurrentExecutingService;
         public Configuration Config => _manager.Config;
         /// <summary>
         /// Returns this shard's connection score.

ShardManager.cs

@@ -27,7 +27,13 @@ namespace BirthdayBot
         /// <summary>
         /// Number of shards allowed to be destroyed before forcing the program to close.
         /// </summary>
-        private const int MaxDestroyedShards = 5;
+        private const int MaxDestroyedShards = 10; // TODO make configurable
+
+        /// <summary>
+        /// Number of concurrent shard startups to happen on each check.
+        /// This value is also used in <see cref="DataRetention"/>.
+        /// </summary>
+        public const int MaxConcurrentOperations = 5;
 
         /// <summary>
         /// Amount of time without a completed background service run before a shard instance
@@ -145,7 +151,7 @@ namespace BirthdayBot
                     Log($"Bot uptime: {Common.BotUptime}");
 
                     // Iterate through shard list, extract data
-                    var guildInfo = new Dictionary<int, (int, int, TimeSpan)>();
+                    var guildInfo = new Dictionary<int, (int, int, TimeSpan, string)>();
                     var now = DateTimeOffset.UtcNow;
                     var nullShards = new List<int>();
                     foreach (var item in _shards)
@@ -160,8 +166,9 @@ namespace BirthdayBot
                         var guildCount = shard.DiscordClient.Guilds.Count;
                         var connScore = shard.ConnectionScore;
                         var lastRun = now - shard.LastBackgroundRun;
+                        var lastExec = shard.CurrentExecutingService ?? "null";
 
-                        guildInfo[item.Key] = (guildCount, connScore, lastRun);
+                        guildInfo[item.Key] = (guildCount, connScore, lastRun, lastExec);
                     }
 
                     // Process info
@@ -203,7 +210,8 @@ namespace BirthdayBot
                             {
                                 result.Remove(result.Length - 1, 1);
                                 result.Append($"[{guildInfo[item].Item2:+0;-0}");
-                                result.Append($" {Math.Floor(guildInfo[item].Item3.TotalSeconds):000}s] ");
+                                result.Append($" {Math.Floor(guildInfo[item].Item3.TotalSeconds):000}s");
+                                result.Append($" {guildInfo[item].Item4}] ");
                             }
                         }
                         if (result.Length > 0) result.Remove(result.Length - 1, 1);
@@ -228,7 +236,7 @@ namespace BirthdayBot
                     else
                     {
                         // Start up any missing shards
-                        int startAllowance = 4;
+                        int startAllowance = MaxConcurrentOperations;
                         foreach (var id in nullShards)
                         {
                             // To avoid possible issues with resources strained over so many shards starting at once,