diff --git a/BackgroundServices/AutoUserDownload.cs b/BackgroundServices/AutoUserDownload.cs index dffcd6f..c4cb47c 100644 --- a/BackgroundServices/AutoUserDownload.cs +++ b/BackgroundServices/AutoUserDownload.cs @@ -8,17 +8,27 @@ namespace BirthdayBot.BackgroundServices; class AutoUserDownload : BackgroundService { public AutoUserDownload(ShardInstance instance) : base(instance) { } + private static readonly HashSet _failedDownloads = new(); + private static readonly TimeSpan _singleDlTimeout = ShardManager.DeadShardThreshold / 3; + public override async Task OnTick(int tickCount, CancellationToken token) { // Take action if a guild's cache is incomplete... - var incompleteCaches = ShardInstance.DiscordClient.Guilds.Where(g => !g.HasAllMembers).Select(g => g.Id).ToHashSet(); + var incompleteCaches = ShardInstance.DiscordClient.Guilds + .Where(g => !g.HasAllMembers) + .Select(g => g.Id) + .ToHashSet(); // ...and if the guild contains any user data IEnumerable mustFetch; try { await DbConcurrentOperationsLock.WaitAsync(token); using var db = new BotDatabaseContext(); - mustFetch = db.UserEntries.AsNoTracking() - .Where(e => incompleteCaches.Contains(e.GuildId)).Select(e => e.GuildId).Distinct() - .ToList(); + lock (_failedDownloads) + mustFetch = db.UserEntries.AsNoTracking() + .Where(e => incompleteCaches.Contains(e.GuildId)) + .Select(e => e.GuildId) + .Distinct() + .Where(e => !_failedDownloads.Contains(e)) + .ToList(); } finally { try { DbConcurrentOperationsLock.Release(); @@ -26,17 +36,31 @@ class AutoUserDownload : BackgroundService { } var processed = 0; + var processStartTime = DateTimeOffset.UtcNow; foreach (var item in mustFetch) { - // May cause a disconnect in certain situations. Cancel all further attempts until the next pass if it happens. + // May cause a disconnect in certain situations. Make no further attempts until the next pass if it happens. if (ShardInstance.DiscordClient.ConnectionState != ConnectionState.Connected) break; - var guild = ShardInstance.DiscordClient.GetGuild((ulong)item); + var guild = ShardInstance.DiscordClient.GetGuild(item); if (guild == null) continue; // A guild disappeared...? - await guild.DownloadUsersAsync().ConfigureAwait(false); // We're already on a seperate thread, no need to use Task.Run - await Task.Delay(200, CancellationToken.None).ConfigureAwait(false); // Must delay, or else it seems to hang... + + await Task.Delay(200, CancellationToken.None); // Delay a bit (reduces the possibility of hanging, somehow). processed++; + var dl = guild.DownloadUsersAsync(); + dl.Wait((int)_singleDlTimeout.TotalMilliseconds / 2, token); + if (dl.IsFaulted) { + Log("Exception thrown by download task: " + dl.Exception); + break; + } else if (!dl.IsCompletedSuccessfully) { + Log($"Task for guild {guild.Id} is unresponsive. Skipping guild. Members: {guild.MemberCount}. Name: {guild.Name}."); + lock (_failedDownloads) _failedDownloads.Add(guild.Id); + continue; + } + + // Prevent unnecessary disconnections by ShardManager if we're taking too long + if (DateTimeOffset.UtcNow - processStartTime > _singleDlTimeout) break; } - if (processed > 25) Log($"Explicit user list request processed for {processed} guild(s)."); + if (processed > 10) Log($"Member list downloads handled for {processed} guilds."); } } diff --git a/BirthdayBot.csproj b/BirthdayBot.csproj index bfda17f..8b6d6bc 100644 --- a/BirthdayBot.csproj +++ b/BirthdayBot.csproj @@ -5,7 +5,7 @@ net6.0 enable enable - 3.4.7 + 3.4.8 NoiTheCat @@ -22,7 +22,7 @@ - + all diff --git a/ShardManager.cs b/ShardManager.cs index ff30a64..2732204 100644 --- a/ShardManager.cs +++ b/ShardManager.cs @@ -25,7 +25,7 @@ class ShardManager : IDisposable { /// Amount of time without a completed background service run before a shard instance /// is considered "dead" and tasked to be removed. /// - private static readonly TimeSpan DeadShardThreshold = new(0, 20, 0); + public static readonly TimeSpan DeadShardThreshold = new(0, 20, 0); /// /// A dictionary with shard IDs as its keys and shard instances as its values.