Update services to use EF queries

BackgroundServices/AutoUserDownload.cs

@@ -9,29 +9,21 @@ class AutoUserDownload : BackgroundService {
     public AutoUserDownload(ShardInstance instance) : base(instance) { }
 
     public override async Task OnTick(int tickCount, CancellationToken token) {
-        foreach (var guild in ShardInstance.DiscordClient.Guilds) {
+        using var db = new BotDatabaseContext();
-            // Has the potential to disconnect while in the middle of processing.
+
+        // Take action if a guild's cache is incomplete...
+        var incompleteCaches = ShardInstance.DiscordClient.Guilds.Where(g => !g.HasAllMembers).Select(g => (long)g.Id).ToHashSet();
+        // ...and if the guild contains any user data
+        var mustFetch = db.UserEntries.Where(e => incompleteCaches.Contains(e.GuildId)).Select(e => e.GuildId).Distinct();
+
+        foreach (var item in mustFetch) {
+            // May cause a disconnect in certain situations. Cancel all further attempts until the next pass if it happens.
             if (ShardInstance.DiscordClient.ConnectionState != ConnectionState.Connected) return;
 
-            // Determine if there is action to be taken...
+            var guild = ShardInstance.DiscordClient.GetGuild((ulong)item);
-            if (!guild.HasAllMembers && await GuildUserAnyAsync(guild.Id).ConfigureAwait(false)) {
+            if (guild == null) continue; // A guild disappeared...?
-                await guild.DownloadUsersAsync().ConfigureAwait(false); // This is already on a separate thread; no need to Task.Run
+            await guild.DownloadUsersAsync().ConfigureAwait(false); // We're already on a seperate thread, no need to use Task.Run
             await Task.Delay(200, CancellationToken.None).ConfigureAwait(false); // Must delay, or else it seems to hang...
         }
     }
 }
-
-    /// <summary>
-    /// Determines if the user database contains any entries corresponding to this guild.
-    /// </summary>
-    /// <returns>True if any entries exist.</returns>
-    private static async Task<bool> GuildUserAnyAsync(ulong guildId) {
-        using var db = await Database.OpenConnectionAsync().ConfigureAwait(false);
-        using var c = db.CreateCommand();
-        c.CommandText = $"select true from {GuildUserConfiguration.BackingTable} where guild_id = @Gid limit 1";
-        c.Parameters.Add("@Gid", NpgsqlTypes.NpgsqlDbType.Bigint).Value = (long)guildId;
-        await c.PrepareAsync(CancellationToken.None).ConfigureAwait(false);
-        using var r = await c.ExecuteReaderAsync(CancellationToken.None).ConfigureAwait(false);
-        return r.Read();
-    }
-}

BackgroundServices/BackgroundService.cs

@@ -1,7 +1,4 @@
-using System.Threading;
+namespace BirthdayBot.BackgroundServices;
-using System.Threading.Tasks;
-
-namespace BirthdayBot.BackgroundServices;
 
 abstract class BackgroundService {
     protected ShardInstance ShardInstance { get; }

BackgroundServices/BirthdayRoleUpdate.cs

@@ -15,62 +15,51 @@ class BirthdayRoleUpdate : BackgroundService {
     /// Processes birthday updates for all available guilds synchronously.
     /// </summary>
     public override async Task OnTick(int tickCount, CancellationToken token) {
-        var exs = new List<Exception>();
+        // For database efficiency, fetch all database information at once before proceeding
-        foreach (var guild in ShardInstance.DiscordClient.Guilds) {
+        // and combine it into the guild IDs that will be processed
+        using var db = new BotDatabaseContext();
+        var shardGuilds = ShardInstance.DiscordClient.Guilds.Select(g => (long)g.Id).ToHashSet();
+        var settings = db.GuildConfigurations.Where(s => shardGuilds.Contains(s.GuildId));
+        var guildChecks = shardGuilds.Join(settings, o => o, i => i.GuildId, (id, conf) => new { Key = (ulong)id, Value = conf });
+
+        var exceptions = new List<Exception>();
+        foreach (var pair in guildChecks) {
+            var guild = ShardInstance.DiscordClient.GetGuild(pair.Key);
+            if (guild == null) continue; // A guild disappeared...?
+            var guildConf = pair.Value;
+
+            // Check task cancellation here. Processing during a single guild is never interrupted.
+            if (token.IsCancellationRequested) throw new TaskCanceledException();
+
             if (ShardInstance.DiscordClient.ConnectionState != Discord.ConnectionState.Connected) {
                 Log("Client is not connected. Stopping early.");
                 return;
             }
 
-            // Check task cancellation here. Processing during a single guild is never interrupted.
-            if (token.IsCancellationRequested) throw new TaskCanceledException();
-
             try {
-                await ProcessGuildAsync(guild).ConfigureAwait(false);
+                // Verify that role settings and permissions are usable
-            } catch (Exception ex) {
+                SocketRole? role = guild.GetRole((ulong)(guildConf.RoleId ?? 0));
-                // Catch all exceptions per-guild but continue processing, throw at end.
-                exs.Add(ex);
-            }
-        }
-        if (exs.Count != 0) throw new AggregateException(exs);
-    }
-
-    /// <summary>
-    /// Main method where actual guild processing occurs.
-    /// </summary>
-    private static async Task ProcessGuildAsync(SocketGuild guild) {
-        // Load guild information - stop if local cache is unavailable.
-        if (!Common.HasMostMembersDownloaded(guild)) return;
-        var gc = await GuildConfiguration.LoadAsync(guild.Id, true).ConfigureAwait(false);
-        if (gc == null) return;
-
-        // Check if role settings are correct before continuing with further processing
-        SocketRole? role = guild.GetRole(gc.RoleId ?? 0);
                 if (role == null || !guild.CurrentUser.GuildPermissions.ManageRoles || role.Position >= guild.CurrentUser.Hierarchy) return;
 
-        // Determine who's currently having a birthday
+                // Load up user configs and begin processing birthdays
-        var users = await GuildUserConfiguration.LoadAllAsync(guild.Id).ConfigureAwait(false);
+                await db.Entry(guildConf).Collection(t => t.UserEntries).LoadAsync(CancellationToken.None);
-        var tz = gc.TimeZone;
+                var birthdays = GetGuildCurrentBirthdays(guildConf.UserEntries, guildConf.TimeZone);
-        var birthdays = GetGuildCurrentBirthdays(users, tz);
                 // Note: Don't quit here if zero people are having birthdays. Roles may still need to be removed by BirthdayApply.
 
-        IEnumerable<SocketGuildUser> announcementList;
                 // Update roles as appropriate
-        try {
+                var announcementList = await UpdateGuildBirthdayRoles(guild, role, birthdays);
-            var updateResult = await UpdateGuildBirthdayRoles(guild, role, birthdays).ConfigureAwait(false);
-            announcementList = updateResult.Item1;
-        } catch (Discord.Net.HttpException) {
-            return;
-        }
 
                 // Birthday announcement
-        var announce = gc.AnnounceMessages;
+                var channel = guild.GetTextChannel((ulong)(guildConf.ChannelAnnounceId ?? 0));
-        var announceping = gc.AnnouncePing;
-        SocketTextChannel? channel = null;
-        if (gc.AnnounceChannelId.HasValue) channel = guild.GetTextChannel(gc.AnnounceChannelId.Value);
                 if (announcementList.Any()) {
-            await AnnounceBirthdaysAsync(announce, announceping, channel, announcementList).ConfigureAwait(false);
+                    await AnnounceBirthdaysAsync(guildConf, channel, announcementList);
                 }
+            } catch (Exception ex) {
+                // Catch all exceptions per-guild but continue processing, throw at end.
+                exceptions.Add(ex);
+            }
+        }
+        if (exceptions.Count != 0) throw new AggregateException(exceptions);
     }
 
     /// <summary>
@@ -82,7 +71,8 @@ class BirthdayRoleUpdate : BackgroundService {
 #pragma warning restore 618
     public static HashSet<ulong> GetGuildCurrentBirthdays(IEnumerable<GuildUserConfiguration> guildUsers, string? defaultTzStr) {
         var tzdb = DateTimeZoneProviders.Tzdb;
-        DateTimeZone defaultTz = (defaultTzStr != null ? DateTimeZoneProviders.Tzdb.GetZoneOrNull(defaultTzStr) : null) ?? tzdb.GetZoneOrNull("UTC")!;
+        DateTimeZone defaultTz = (defaultTzStr != null ? DateTimeZoneProviders.Tzdb.GetZoneOrNull(defaultTzStr) : null)
+            ?? tzdb.GetZoneOrNull("UTC")!;
 
         var birthdayUsers = new HashSet<ulong>();
         foreach (var item in guildUsers) {
@@ -138,11 +128,9 @@ class BirthdayRoleUpdate : BackgroundService {
     /// Sets the birthday role to all applicable users. Unsets it from all others who may have it.
     /// </summary>
     /// <returns>
-    /// First item: List of users who had the birthday role applied, used to announce.
+    /// List of users who had the birthday role applied, used to announce.
-    /// Second item: Counts of users who have had roles added/removed, used for operation reporting.
     /// </returns>
-    private static async Task<(IEnumerable<SocketGuildUser>, (int, int))> UpdateGuildBirthdayRoles(
+    private static async Task<IEnumerable<SocketGuildUser>> UpdateGuildBirthdayRoles(SocketGuild g, SocketRole r, HashSet<ulong> names) {
-        SocketGuild g, SocketRole r, HashSet<ulong> names) {
         // Check members currently with the role. Figure out which users to remove it from.
         var roleRemoves = new List<SocketGuildUser>();
         var roleKeeps = new HashSet<ulong>();
@@ -165,7 +153,7 @@ class BirthdayRoleUpdate : BackgroundService {
             newBirthdays.Add(member);
         }
 
-        return (newBirthdays, (newBirthdays.Count, roleRemoves.Count));
+        return newBirthdays;
     }
 
     public const string DefaultAnnounce = "Please wish a happy birthday to %n!";
@@ -174,21 +162,20 @@ class BirthdayRoleUpdate : BackgroundService {
     /// <summary>
     /// Attempts to send an announcement message.
     /// </summary>
-    private static async Task AnnounceBirthdaysAsync(
+    private static async Task AnnounceBirthdaysAsync(GuildConfig settings, SocketTextChannel? c, IEnumerable<SocketGuildUser> names) {
-        (string?, string?) announce, bool announcePing, SocketTextChannel? c, IEnumerable<SocketGuildUser> names) {
         if (c == null) return;
         if (!c.Guild.CurrentUser.GetPermissions(c).SendMessages) return;
 
         string announceMsg;
-        if (names.Count() == 1) announceMsg = announce.Item1 ?? announce.Item2 ?? DefaultAnnounce;
+        if (names.Count() == 1) announceMsg = settings.AnnounceMessage ?? settings.AnnounceMessagePl ?? DefaultAnnounce;
-        else announceMsg = announce.Item2 ?? announce.Item1 ?? DefaultAnnouncePl;
+        else announceMsg = settings.AnnounceMessagePl ?? settings.AnnounceMessage ?? DefaultAnnouncePl;
         announceMsg = announceMsg.TrimEnd();
         if (!announceMsg.Contains("%n")) announceMsg += " %n";
 
         // Build sorted name list
         var namestrings = new List<string>();
         foreach (var item in names)
-            namestrings.Add(Common.FormatName(item, announcePing));
+            namestrings.Add(Common.FormatName(item, settings.AnnouncePing));
         namestrings.Sort(StringComparer.OrdinalIgnoreCase);
 
         var namedisplay = new StringBuilder();

BackgroundServices/DataRetention.cs

@@ -1,5 +1,4 @@
 using BirthdayBot.Data;
-using NpgsqlTypes;
 using System.Text;
 
 namespace BirthdayBot.BackgroundServices;
@@ -20,96 +19,49 @@ class DataRetention : BackgroundService {
         // On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks.
         if ((tickCount + ShardInstance.ShardId) % ProcessInterval != 0) return;
 
-        try {
+        using var db = new BotDatabaseContext();
-            // A semaphore is used to restrict this work being done concurrently on other shards
+        var now = DateTimeOffset.UtcNow;
-            // to avoid putting pressure on the SQL connection pool. Clearing old database information
+        int updatedGuilds = 0, updatedUsers = 0;
-            // ultimately is a low priority among other tasks.
-            await _updateLock.WaitAsync(token).ConfigureAwait(false);
-        } catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException) {
-            // Caller does not expect the exception that SemaphoreSlim throws...
-            throw new TaskCanceledException();
-        }
-        try {
-            // Build a list of all values across all guilds to update
-            var updateList = new Dictionary<ulong, List<ulong>>();
-            foreach (var g in ShardInstance.DiscordClient.Guilds) {
-                // Get list of IDs for all users who exist in the database and currently exist in the guild
-                var userList = GuildUserConfiguration.LoadAllAsync(g.Id);
-                var guildUserIds = from gu in g.Users select gu.Id;
-                var savedUserIds = from cu in await userList.ConfigureAwait(false) select cu.UserId;
-                var existingCachedIds = savedUserIds.Intersect(guildUserIds);
-                updateList[g.Id] = existingCachedIds.ToList();
-            }
 
-            using var db = await Database.OpenConnectionAsync().ConfigureAwait(false);
+        foreach (var guild in ShardInstance.DiscordClient.Guilds) {
+            // Update guild, fetch users from database
+            var dbGuild = db.GuildConfigurations.Where(s => s.GuildId == (long)guild.Id).FirstOrDefault();
+            if (dbGuild == null) continue;
+            dbGuild.LastSeen = now;
+            updatedGuilds++;
 
-            // Statement for updating last_seen in guilds
+            // Update users
-            var cUpdateGuild = db.CreateCommand();
+            var localIds = guild.Users.Select(u => (long)u.Id);
-            cUpdateGuild.CommandText = $"update {GuildConfiguration.BackingTable} set last_seen = now() "
+            var dbSavedIds = db.UserEntries.Where(e => e.GuildId == (long)guild.Id).Select(e => e.UserId);
-                + "where guild_id = @Gid";
+            var usersToUpdate = localIds.Intersect(dbSavedIds).ToHashSet();
-            var pUpdateG = cUpdateGuild.Parameters.Add("@Gid", NpgsqlDbType.Bigint);
+            foreach (var user in db.UserEntries.Where(e => e.GuildId == (long)guild.Id && usersToUpdate.Contains(e.UserId))) {
-            cUpdateGuild.Prepare();
+                user.LastSeen = now;
-
+                updatedUsers++;
-            // Statement for updating last_seen in guild users
-            var cUpdateGuildUser = db.CreateCommand();
-            cUpdateGuildUser.CommandText = $"update {GuildUserConfiguration.BackingTable} set last_seen = now() "
-                + "where guild_id = @Gid and user_id = @Uid";
-            var pUpdateGU_g = cUpdateGuildUser.Parameters.Add("@Gid", NpgsqlDbType.Bigint);
-            var pUpdateGU_u = cUpdateGuildUser.Parameters.Add("@Uid", NpgsqlDbType.Bigint);
-            cUpdateGuildUser.Prepare();
-
-            // Do actual updates
-            int updatedGuilds = 0;
-            int updatedUsers = 0;
-            using (var tUpdate = db.BeginTransaction()) {
-                foreach (var item in updateList) {
-                    var guild = item.Key;
-                    var userlist = item.Value;
-
-                    pUpdateG.Value = (long)guild;
-                    updatedGuilds += await cUpdateGuild.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
-
-                    pUpdateGU_g.Value = (long)guild;
-                    foreach (var userid in userlist) {
-                        pUpdateGU_u.Value = (long)userid;
-                        updatedUsers += await cUpdateGuildUser.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
             }
         }
-                await tUpdate.CommitAsync(CancellationToken.None).ConfigureAwait(false);
+
-            }
+        // And let go of old data
+        var staleGuilds = db.GuildConfigurations.Where(s => now - TimeSpan.FromDays(StaleGuildThreshold) > s.LastSeen);
+        var staleUsers = db.UserEntries.Where(e => now - TimeSpan.FromDays(StaleUserThreashold) > e.LastSeen);
+        int staleGuildCount = staleGuilds.Count(), staleUserCount = staleUsers.Count();
+        db.GuildConfigurations.RemoveRange(staleGuilds);
+        db.UserEntries.RemoveRange(staleUsers);
+
+        await db.SaveChangesAsync(CancellationToken.None);
+            
         var resultText = new StringBuilder();
         resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users.");
-
+        if (staleGuildCount != 0 || staleUserCount != 0) {
-            // Deletes both guild and user data if it hasn't been seen for over the threshold defined at the top of this file
-            // Expects referencing tables to have 'on delete cascade'
-            int staleGuilds, staleUsers;
-            using (var tRemove = db.BeginTransaction()) {
-                using (var c = db.CreateCommand()) {
-                    c.CommandText = $"delete from {GuildConfiguration.BackingTable}" +
-                        $" where (now() - interval '{StaleGuildThreshold} days') > last_seen";
-                    staleGuilds = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
-                }
-                using (var c = db.CreateCommand()) {
-                    c.CommandText = $"delete from {GuildUserConfiguration.BackingTable}" +
-                        $" where (now() - interval '{StaleUserThreashold} days') > last_seen";
-                    staleUsers = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
-                }
-                await tRemove.CommitAsync(CancellationToken.None).ConfigureAwait(false);
-            }
-            if (staleGuilds != 0 || staleUsers != 0) {
             resultText.Append(" Discarded ");
-                if (staleGuilds != 0) {
+            if (staleGuildCount != 0) {
-                    resultText.Append($"{staleGuilds} guilds");
+                resultText.Append($"{staleGuildCount} guilds");
-                    if (staleUsers != 0) resultText.Append(", ");
+                if (staleUserCount != 0) resultText.Append(", ");
             }
-                if (staleUsers != 0) {
+            if (staleUserCount != 0) {
-                    resultText.Append($"{staleUsers} users");
+                resultText.Append($"{staleUserCount} users");
             }
             resultText.Append('.');
         }
         Log(resultText.ToString());
-        } finally {
-            _updateLock.Release();
-        }
     }
 }

BackgroundServices/ExternalStatisticsReporting.cs

@@ -1,8 +1,4 @@
-using System;
+using System.Text;
-using System.Net.Http;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
 
 namespace BirthdayBot.BackgroundServices;
 
@@ -11,7 +7,7 @@ namespace BirthdayBot.BackgroundServices;
 /// </summary>
 class ExternalStatisticsReporting : BackgroundService {
     const int ProcessInterval = 1200 / ShardBackgroundWorker.Interval; // Process every ~20 minutes
-    const int ProcessOffset = 300 / ShardBackgroundWorker.Interval; // Begin processing 5 minutes after shard start
+    const int ProcessOffset = 300 / ShardBackgroundWorker.Interval; // Begin processing ~5 minutes after shard start
 
     private static readonly HttpClient _httpClient = new();
 

Data/BotDatabaseContext.cs

@@ -37,7 +37,8 @@ public class BotDatabaseContext : DbContext {
             entity.HasOne(d => d.Guild)
                 .WithMany(p => p.BlockedUsers)
                 .HasForeignKey(d => d.GuildId)
-                .HasConstraintName("banned_users_guild_id_fkey");
+                .HasConstraintName("banned_users_guild_id_fkey")
+                .OnDelete(DeleteBehavior.Cascade);
         });
 
         modelBuilder.Entity<GuildConfig>(entity => {
@@ -58,7 +59,8 @@ public class BotDatabaseContext : DbContext {
             entity.HasOne(d => d.Guild)
                 .WithMany(p => p.UserEntries)
                 .HasForeignKey(d => d.GuildId)
-                .HasConstraintName("user_birthdays_guild_id_fkey");
+                .HasConstraintName("user_birthdays_guild_id_fkey")
+                .OnDelete(DeleteBehavior.Cascade);
         });
     }
 }