Add some leeway regarding incomplete user cache

Implemented several workarounds to what appearto be a library bug in
which not all users will be downloaded onto the cache. User cache
checking and downloading is delegated to a new Common method and a
new BackgroundService, respectively.
This is hopefully temporary.

BackgroundServices/BirthdayRoleUpdate.cs

@@ -74,11 +74,6 @@ namespace BirthdayBot.BackgroundServices
             if (diag.RoleCheck != null) return diag;
 
             // Determine who's currently having a birthday
-            if (!guild.HasAllMembers)
-            {
-                await guild.DownloadUsersAsync().ConfigureAwait(false);
-                await Task.Delay(500);
-            }
             var users = await GuildUserConfiguration.LoadAllAsync(guild.Id).ConfigureAwait(false);
             var tz = gc.TimeZone;
             var birthdays = GetGuildCurrentBirthdays(users, tz);

BackgroundServices/SelectiveAutoUserDownload.cs

@@ -0,0 +1,86 @@
+using BirthdayBot.Data;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace BirthdayBot.BackgroundServices
+{
+    /// <summary>
+    /// A type of workaround to the issue of user information not being cached for guilds that
+    /// have user information existing in the bot's database. This service runs frequently and
+    /// determines guilds in which user data must be downloaded, and proceeds to request it.
+    /// </summary>
+    class SelectiveAutoUserDownload : BackgroundService
+    {
+        private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(2);
+
+        private readonly HashSet<ulong> _fetchRequests = new HashSet<ulong>();
+
+        public SelectiveAutoUserDownload(ShardInstance instance) : base(instance) { }
+
+        public override async Task OnTick(CancellationToken token)
+        {
+            IEnumerable<ulong> requests;
+            lock (_fetchRequests)
+            {
+                requests = _fetchRequests.ToArray();
+                _fetchRequests.Clear();
+            }
+
+            foreach (var guild in ShardInstance.DiscordClient.Guilds)
+            {
+                if (ShardInstance.DiscordClient.ConnectionState != Discord.ConnectionState.Connected)
+                {
+                    Log("Client no longer connected. Stopping early.");
+                    return;
+                }
+
+                // Determine if there is action to be taken...
+                if (guild.HasAllMembers) continue;
+                if (requests.Contains(guild.Id) || await GuildUserAnyAsync(guild.Id, token).ConfigureAwait(false))
+                {
+                    await guild.DownloadUsersAsync().ConfigureAwait(false);
+                    await Task.Delay(500).ConfigureAwait(false);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Determines if the user database contains any entries corresponding to this guild.
+        /// </summary>
+        /// <returns>True if any entries exist.</returns>
+        private async Task<bool> GuildUserAnyAsync(ulong guildId, CancellationToken token)
+        {
+            try
+            {
+                await _updateLock.WaitAsync(token).ConfigureAwait(false);
+            }
+            catch (Exception ex) when (ex is OperationCanceledException || ex is ObjectDisposedException)
+            {
+                // Calling thread does not expect the exception that SemaphoreSlim throws...
+                throw new TaskCanceledException();
+            }
+            try
+            {
+                using var db = await Database.OpenConnectionAsync().ConfigureAwait(false);
+                using var c = db.CreateCommand();
+                c.CommandText = $"select count(*) from {GuildUserConfiguration.BackingTable} where guild_id = @Gid";
+                c.Parameters.Add("@Gid", NpgsqlTypes.NpgsqlDbType.Bigint).Value = (long)guildId;
+                await c.PrepareAsync().ConfigureAwait(false);
+                var r = (long)await c.ExecuteScalarAsync(token).ConfigureAwait(false);
+                return r != 0;
+            }
+            finally
+            {
+                _updateLock.Release();
+            }
+        }
+
+        public void RequestDownload(ulong guildId)
+        {
+            lock (_fetchRequests) _fetchRequests.Add(guildId);
+        }
+    }
+}

BackgroundServices/ShardBackgroundWorker.cs

@@ -23,6 +23,7 @@ namespace BirthdayBot.BackgroundServices
 
         public ConnectionStatus ConnStatus { get; }
         public BirthdayRoleUpdate BirthdayUpdater { get; }
+        public SelectiveAutoUserDownload UserDownloader { get; }
         public DateTimeOffset LastBackgroundRun { get; private set; }
         public int ConnectionScore => ConnStatus.Score;
 
@@ -33,8 +34,10 @@ namespace BirthdayBot.BackgroundServices
 
             ConnStatus = new ConnectionStatus(instance);
             BirthdayUpdater = new BirthdayRoleUpdate(instance);
+            UserDownloader = new SelectiveAutoUserDownload(instance);
             _workers = new List<BackgroundService>()
             {
+                {UserDownloader},
                 {BirthdayUpdater},
                 {new DataRetention(instance)}
             };

BirthdayBot.csproj

@@ -1,9 +1,9 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>netcoreapp3.1</TargetFramework>
-    <Version>3.0.0</Version>
+    <Version>3.0.2</Version>
     <PackageId>BirthdayBot</PackageId>
     <Authors>NoiTheCat</Authors>
     <Product>BirthdayBot</Product>
@@ -19,7 +19,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Discord.Net" Version="2.3.0-dev-20201028.4" />
+    <PackageReference Include="Discord.Net" Version="2.3.0" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="NodaTime" Version="3.0.3" />
     <PackageReference Include="Npgsql" Version="4.1.5" />

Common.cs

@@ -43,5 +43,26 @@ namespace BirthdayBot
         };
 
         public static string BotUptime => (DateTimeOffset.UtcNow - Program.BotStartTime).ToString("d' days, 'hh':'mm':'ss");
+
+        /// <summary>
+        /// An alternative to <see cref="SocketGuild.HasAllMembers"/>.
+        /// Returns true if *most* members have been downloaded.
+        /// </summary>
+        public static bool HasMostMembersDownloaded(SocketGuild guild)
+        {
+            if (guild.HasAllMembers) return true;
+            if (guild.MemberCount > 30)
+            {
+                // For guilds of size over 30, require 85% or more of the members to be known
+                // (26/30, 42/50, 255/300, etc)
+                int threshold = (int)(guild.MemberCount * 0.85);
+                return guild.DownloadedMemberCount >= threshold;
+            }
+            else
+            {
+                // For smaller guilds, fail if two or more members are missing
+                return guild.MemberCount - guild.DownloadedMemberCount <= 2;
+            }
+        }
     }
 }

ShardInstance.cs

@@ -107,6 +107,8 @@ namespace BirthdayBot
         public Task<string> ForceBirthdayUpdateAsync(SocketGuild guild)
             => _background.BirthdayUpdater.SingleProcessGuildAsync(guild);
 
+        public void RequestDownloadUsers(ulong guildId) => _background.UserDownloader.RequestDownload(guildId);
+
         #region Event handling
         private Task Client_Log(LogMessage arg)
         {

UserInterface/CommandsCommon.cs

@@ -22,6 +22,7 @@ namespace BirthdayBot.UserInterface
         public const string ParameterError = ":x: Invalid usage. Refer to how to use the command and try again.";
         public const string NoParameterError = ":x: This command does not accept any parameters.";
         public const string InternalError = ":x: An internal bot error occurred. The bot maintainer has been notified of the issue.";
+        public const string UsersNotDownloadedError = ":x: Currently unavailable. Please try again in a few minutes.";
 
         public delegate Task CommandHandler(ShardInstance instance, GuildConfiguration gconf,
                                             string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser);

UserInterface/ListingCommands.cs

@@ -38,6 +38,13 @@ namespace BirthdayBot.UserInterface
         private async Task CmdWhen(ShardInstance instance, GuildConfiguration gconf,
                                    string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser)
         {
+            if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
+            {
+                instance.RequestDownloadUsers(reqChannel.Guild.Id);
+                await reqChannel.SendMessageAsync(UsersNotDownloadedError);
+                return;
+            }
+
             // Requires a parameter
             if (param.Length == 1)
             {
@@ -103,6 +110,13 @@ namespace BirthdayBot.UserInterface
                 return;
             }
 
+            if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
+            {
+                instance.RequestDownloadUsers(reqChannel.Guild.Id);
+                await reqChannel.SendMessageAsync(UsersNotDownloadedError);
+                return;
+            }
+
             bool useCsv = false;
             // Check for CSV option
             if (param.Length == 2)
@@ -162,6 +176,13 @@ namespace BirthdayBot.UserInterface
         private async Task CmdUpcoming(ShardInstance instance, GuildConfiguration gconf,
                                        string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser)
         {
+            if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
+            {
+                instance.RequestDownloadUsers(reqChannel.Guild.Id);
+                await reqChannel.SendMessageAsync(UsersNotDownloadedError);
+                return;
+            }
+
             var now = DateTimeOffset.UtcNow;
             var search = DateIndex(now.Month, now.Day) - 8; // begin search 8 days prior to current date UTC
             if (search <= 0) search = 366 - Math.Abs(search);

UserInterface/ManagerCommands.cs

@@ -393,6 +393,13 @@ namespace BirthdayBot.UserInterface
             // Moderators only. As with config, silently drop if this check fails.
             if (!gconf.IsBotModerator(reqUser)) return;
 
+            if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
+            {
+                instance.RequestDownloadUsers(reqChannel.Guild.Id);
+                await reqChannel.SendMessageAsync(UsersNotDownloadedError);
+                return;
+            }
+
             if (param.Length != 3)
             {
                 await reqChannel.SendMessageAsync(ParameterError, embed: DocOverride.UsageEmbed).ConfigureAwait(false);
@@ -462,7 +469,10 @@ namespace BirthdayBot.UserInterface
 
             try
             {
-                var result = await instance.ForceBirthdayUpdateAsync(reqChannel.Guild).ConfigureAwait(false);
+                var guild = reqChannel.Guild;
+                string result = $"\nServer ID: {guild.Id} | Bot shard ID: {instance.ShardId:00}";
+                result += $"\nLocally cached members: {guild.DownloadedMemberCount} out of {guild.MemberCount}";
+                result += "\n" + await instance.ForceBirthdayUpdateAsync(guild).ConfigureAwait(false);
                 await reqChannel.SendMessageAsync(result).ConfigureAwait(false);
             }
             catch (Exception ex)