From 9ef164bac1b80b17e79f7b4d8a2e854d108842f3 Mon Sep 17 00:00:00 2001 From: Noi Date: Fri, 9 Oct 2020 23:48:43 -0700 Subject: [PATCH] Update DataRetention service Renamed from StaleDataCleaner. Modified to run only once concurrently, to not put extra load on the database connection pool. --- BackgroundServices/DataRetention.cs | 124 ++++++++++++++++++++ BackgroundServices/ShardBackgroundWorker.cs | 2 +- BackgroundServices/StaleDataCleaner.cs | 94 --------------- 3 files changed, 125 insertions(+), 95 deletions(-) create mode 100644 BackgroundServices/DataRetention.cs delete mode 100644 BackgroundServices/StaleDataCleaner.cs diff --git a/BackgroundServices/DataRetention.cs b/BackgroundServices/DataRetention.cs new file mode 100644 index 0000000..dfefc0a --- /dev/null +++ b/BackgroundServices/DataRetention.cs @@ -0,0 +1,124 @@ +using BirthdayBot.Data; +using NpgsqlTypes; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace BirthdayBot.BackgroundServices +{ + /// + /// Data retention adherence service: + /// Automatically removes database information for guilds that have not been accessed in a long time. + /// + class DataRetention : BackgroundService + { + private static readonly SemaphoreSlim _updateLock = new SemaphoreSlim(1); + const int ProcessInterval = 600 / ShardBackgroundWorker.Interval; // Process every ~10 minutes + private int _tickCount = 0; + + public DataRetention(ShardInstance instance) : base(instance) { } + + public override async Task OnTick(CancellationToken token) + { + if (++_tickCount % ProcessInterval != 0) + { + // Do not process on every tick. + return; + } + + // A semaphore is used to restrict this work being done concurrently on other shards + // to avoid putting pressure on the SQL connection pool. Updating this is a low priority. + await _updateLock.WaitAsync(token).ConfigureAwait(false); + try + { + // Build a list of all values to update + var updateList = new Dictionary>(); + foreach (var g in ShardInstance.DiscordClient.Guilds) + { + // Get list of IDs for all users who exist in the database and currently exist in the guild + var userList = GuildUserConfiguration.LoadAllAsync(g.Id); + var guildUserIds = from gu in g.Users select gu.Id; + var savedUserIds = from cu in await userList.ConfigureAwait(false) select cu.UserId; + var existingCachedIds = savedUserIds.Intersect(guildUserIds); + updateList[g.Id] = existingCachedIds.ToList(); + } + + using var db = await Database.OpenConnectionAsync().ConfigureAwait(false); + + // Statement for updating last_seen in guilds + var cUpdateGuild = db.CreateCommand(); + cUpdateGuild.CommandText = $"update {GuildConfiguration.BackingTable} set last_seen = now() " + + "where guild_id = @Gid"; + var pUpdateG = cUpdateGuild.Parameters.Add("@Gid", NpgsqlDbType.Bigint); + cUpdateGuild.Prepare(); + + // Statement for updating last_seen in guild users + var cUpdateGuildUser = db.CreateCommand(); + cUpdateGuildUser.CommandText = $"update {GuildUserConfiguration.BackingTable} set last_seen = now() " + + "where guild_id = @Gid and user_id = @Uid"; + var pUpdateGU_g = cUpdateGuildUser.Parameters.Add("@Gid", NpgsqlDbType.Bigint); + var pUpdateGU_u = cUpdateGuildUser.Parameters.Add("@Uid", NpgsqlDbType.Bigint); + cUpdateGuildUser.Prepare(); + + // Do actual updates + int updatedGuilds = 0; + int updatedUsers = 0; + foreach (var item in updateList) + { + var guild = item.Key; + var userlist = item.Value; + + pUpdateG.Value = (long)guild; + updatedGuilds += await cUpdateGuild.ExecuteNonQueryAsync().ConfigureAwait(false); + + pUpdateGU_g.Value = (long)guild; + foreach (var userid in userlist) + { + pUpdateGU_u.Value = (long)userid; + updatedUsers += await cUpdateGuildUser.ExecuteNonQueryAsync().ConfigureAwait(false); + } + } + var resultText = new StringBuilder(); + resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users."); + + // Delete all old values - expects referencing tables to have 'on delete cascade' + using var t = db.BeginTransaction(); + int staleGuilds, staleUsers; + using (var c = db.CreateCommand()) + { + // Delete data for guilds not seen in 4 weeks + c.CommandText = $"delete from {GuildConfiguration.BackingTable} where (now() - interval '28 days') > last_seen"; + staleGuilds = await c.ExecuteNonQueryAsync().ConfigureAwait(false); + } + using (var c = db.CreateCommand()) + { + // Delete data for users not seen in 8 weeks + c.CommandText = $"delete from {GuildUserConfiguration.BackingTable} where (now() - interval '56 days') > last_seen"; + staleUsers = await c.ExecuteNonQueryAsync().ConfigureAwait(false); + } + if (staleGuilds != 0 || staleUsers != 0) + { + resultText.Append(" Discarded "); + if (staleGuilds != 0) + { + resultText.Append($"{staleGuilds} guilds"); + if (staleUsers != 0) resultText.Append(", "); + } + if (staleUsers != 0) + { + resultText.Append($"{staleUsers} standalone users"); + } + resultText.Append('.'); + } + t.Commit(); + Log(resultText.ToString()); + } + finally + { + _updateLock.Release(); + } + } + } +} diff --git a/BackgroundServices/ShardBackgroundWorker.cs b/BackgroundServices/ShardBackgroundWorker.cs index 9640112..2184cf6 100644 --- a/BackgroundServices/ShardBackgroundWorker.cs +++ b/BackgroundServices/ShardBackgroundWorker.cs @@ -36,7 +36,7 @@ namespace BirthdayBot.BackgroundServices _workers = new List() { {BirthdayUpdater}, - {new StaleDataCleaner(instance)} + {new DataRetention(instance)} }; _workerTask = Task.Factory.StartNew(WorkerLoop, _workerCanceller.Token); diff --git a/BackgroundServices/StaleDataCleaner.cs b/BackgroundServices/StaleDataCleaner.cs deleted file mode 100644 index c77ffe7..0000000 --- a/BackgroundServices/StaleDataCleaner.cs +++ /dev/null @@ -1,94 +0,0 @@ -using BirthdayBot.Data; -using NpgsqlTypes; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace BirthdayBot.BackgroundServices -{ - /// - /// Automatically removes database information for guilds that have not been accessed in a long time. - /// - class StaleDataCleaner : BackgroundService - { - private int _tickCount = 0; - - public StaleDataCleaner(ShardInstance instance) : base(instance) { } - - public override async Task OnTick(CancellationToken token) - { - if (++_tickCount % 20 != 0) - { - // Do not process on every tick. - return; - } - - // Build a list of all values to update - var updateList = new Dictionary>(); - foreach (var g in ShardInstance.DiscordClient.Guilds) - { - // Get list of IDs for all users who exist in the database and currently exist in the guild - var savedUserIds = from cu in await GuildUserConfiguration.LoadAllAsync(g.Id) select cu.UserId; - var guildUserIds = from gu in g.Users select gu.Id; - var existingCachedIds = savedUserIds.Intersect(guildUserIds); - updateList[g.Id] = new List(existingCachedIds); - } - - using var db = await Database.OpenConnectionAsync(); - - // Statement for updating last_seen in guilds - var cUpdateGuild = db.CreateCommand(); - cUpdateGuild.CommandText = $"update {GuildConfiguration.BackingTable} set last_seen = now() " - + "where guild_id = @Gid"; - var pUpdateG = cUpdateGuild.Parameters.Add("@Gid", NpgsqlDbType.Bigint); - cUpdateGuild.Prepare(); - - // Statement for updating last_seen in guild users - var cUpdateGuildUser = db.CreateCommand(); - cUpdateGuildUser.CommandText = $"update {GuildUserConfiguration.BackingTable} set last_seen = now() " - + "where guild_id = @Gid and user_id = @Uid"; - var pUpdateGU_g = cUpdateGuildUser.Parameters.Add("@Gid", NpgsqlDbType.Bigint); - var pUpdateGU_u = cUpdateGuildUser.Parameters.Add("@Uid", NpgsqlDbType.Bigint); - cUpdateGuildUser.Prepare(); - - // Do actual updates - int updatedGuilds = 0; - int updatedUsers = 0; - foreach (var item in updateList) - { - var guild = item.Key; - var userlist = item.Value; - - pUpdateG.Value = (long)guild; - updatedGuilds += await cUpdateGuild.ExecuteNonQueryAsync(token); - - pUpdateGU_g.Value = (long)guild; - foreach (var userid in userlist) - { - pUpdateGU_u.Value = (long)userid; - updatedUsers += await cUpdateGuildUser.ExecuteNonQueryAsync(token); - } - } - Log($"Updated last-seen records: {updatedGuilds} guilds, {updatedUsers} users"); - - // Delete all old values - expects referencing tables to have 'on delete cascade' - using var t = db.BeginTransaction(); - int staleGuilds, staleUsers; - using (var c = db.CreateCommand()) - { - // Delete data for guilds not seen in 4 weeks - c.CommandText = $"delete from {GuildConfiguration.BackingTable} where (now() - interval '28 days') > last_seen"; - staleGuilds = c.ExecuteNonQuery(); - } - using (var c = db.CreateCommand()) - { - // Delete data for users not seen in 8 weeks - c.CommandText = $"delete from {GuildUserConfiguration.BackingTable} where (now() - interval '56 days') > last_seen"; - staleUsers = c.ExecuteNonQuery(); - } - Log($"Will remove {staleGuilds} guilds, {staleUsers} users."); - t.Commit(); - } - } -}