Increase threshold, no longer stagger DataRetention

This staggering was most likely the cause of a major amount of data lost recently, in which certain guilds never had their values updated.
The staggering was meant to attempt to reduce load on a server with limited capabilities, and testing shows that it runs on more capable hardware without this issue when run this way.
This commit is contained in:
Noi 2021-11-22 11:49:55 -08:00
parent 67f78e068e
commit f8350fed53
3 changed files with 19 additions and 19 deletions

View file

@ -1,11 +1,6 @@
using BirthdayBot.Data; using BirthdayBot.Data;
using NpgsqlTypes; using NpgsqlTypes;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace BirthdayBot.BackgroundServices; namespace BirthdayBot.BackgroundServices;
@ -14,14 +9,16 @@ namespace BirthdayBot.BackgroundServices;
/// </summary> /// </summary>
class DataRetention : BackgroundService { class DataRetention : BackgroundService {
private static readonly SemaphoreSlim _updateLock = new(ShardManager.MaxConcurrentOperations); private static readonly SemaphoreSlim _updateLock = new(ShardManager.MaxConcurrentOperations);
const int ProcessInterval = 3600 / ShardBackgroundWorker.Interval; // Process about once per hour const int ProcessInterval = 5400 / ShardBackgroundWorker.Interval; // Process about once per hour and a half
const int Stagger = 3; // How many ticks in between each group of guilds to stagger processing. // Amount of days without updates before data is considered stale and up for deletion.
const int StaleGuildThreshold = 180;
const int StaleUserThreashold = 360;
public DataRetention(ShardInstance instance) : base(instance) { } public DataRetention(ShardInstance instance) : base(instance) { }
public override async Task OnTick(int tickCount, CancellationToken token) { public override async Task OnTick(int tickCount, CancellationToken token) {
// On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks. // On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks.
if ((tickCount + ShardInstance.ShardId * Stagger) % ProcessInterval != 0) return; if ((tickCount + ShardInstance.ShardId) % ProcessInterval != 0) return;
try { try {
// A semaphore is used to restrict this work being done concurrently on other shards // A semaphore is used to restrict this work being done concurrently on other shards
@ -33,7 +30,7 @@ class DataRetention : BackgroundService {
throw new TaskCanceledException(); throw new TaskCanceledException();
} }
try { try {
// Build a list of all values to update // Build a list of all values across all guilds to update
var updateList = new Dictionary<ulong, List<ulong>>(); var updateList = new Dictionary<ulong, List<ulong>>();
foreach (var g in ShardInstance.DiscordClient.Guilds) { foreach (var g in ShardInstance.DiscordClient.Guilds) {
// Get list of IDs for all users who exist in the database and currently exist in the guild // Get list of IDs for all users who exist in the database and currently exist in the guild
@ -80,17 +77,18 @@ class DataRetention : BackgroundService {
var resultText = new StringBuilder(); var resultText = new StringBuilder();
resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users."); resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users.");
// Delete all old values - expects referencing tables to have 'on delete cascade' // Deletes both guild and user data if it hasn't been seen for over the threshold defined at the top of this file
// Expects referencing tables to have 'on delete cascade'
using var t = db.BeginTransaction(); using var t = db.BeginTransaction();
int staleGuilds, staleUsers; int staleGuilds, staleUsers;
using (var c = db.CreateCommand()) { using (var c = db.CreateCommand()) {
// Delete data for guilds not seen in 4 weeks c.CommandText = $"delete from {GuildConfiguration.BackingTable}" +
c.CommandText = $"delete from {GuildConfiguration.BackingTable} where (now() - interval '28 days') > last_seen"; $" where (now() - interval '{StaleGuildThreshold} days') > last_seen";
staleGuilds = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); staleGuilds = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
} }
using (var c = db.CreateCommand()) { using (var c = db.CreateCommand()) {
// Delete data for users not seen in 8 weeks c.CommandText = $"delete from {GuildUserConfiguration.BackingTable}" +
c.CommandText = $"delete from {GuildUserConfiguration.BackingTable} where (now() - interval '56 days') > last_seen"; $" where (now() - interval '{StaleUserThreashold} days') > last_seen";
staleUsers = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false); staleUsers = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
} }
if (staleGuilds != 0 || staleUsers != 0) { if (staleGuilds != 0 || staleUsers != 0) {
@ -100,7 +98,7 @@ class DataRetention : BackgroundService {
if (staleUsers != 0) resultText.Append(", "); if (staleUsers != 0) resultText.Append(", ");
} }
if (staleUsers != 0) { if (staleUsers != 0) {
resultText.Append($"{staleUsers} standalone users"); resultText.Append($"{staleUsers} users");
} }
resultText.Append('.'); resultText.Append('.');
} }

View file

@ -80,12 +80,14 @@ class ShardInstance : IDisposable {
public void Log(string source, string message) => Program.Log($"Shard {ShardId:00}] [{source}", message); public void Log(string source, string message) => Program.Log($"Shard {ShardId:00}] [{source}", message);
public void RequestDownloadUsers(ulong guildId) => _background.UserDownloader.RequestDownload(guildId);
#region Event handling #region Event handling
private Task Client_Log(LogMessage arg) { private Task Client_Log(LogMessage arg) {
// TODO revise this some time, filters might need to be modified by now // TODO revise this some time, filters might need to be modified by now
// Suppress certain messages // Suppress certain messages
if (arg.Message != null) { if (arg.Message != null) {
if (arg.Message.StartsWith("Unknown Dispatch ") || arg.Message.StartsWith("Missing Channel")) return Task.CompletedTask; if (arg.Message.StartsWith("Unknown Dispatch ") || arg.Message.StartsWith("Unknown Channel")) return Task.CompletedTask;
switch (arg.Message) // Connection status messages replaced by ShardManager's output switch (arg.Message) // Connection status messages replaced by ShardManager's output
{ {
case "Connecting": case "Connecting":

View file

@ -15,10 +15,10 @@ class ShardManager : IDisposable {
/// <summary> /// <summary>
/// Number of seconds between each time the status task runs, in seconds. /// Number of seconds between each time the status task runs, in seconds.
/// </summary> /// </summary>
private const int StatusInterval = 90; private const int StatusInterval = 60;
/// <summary> /// <summary>
/// Number of shards allowed to be destroyed before forcing the program to close. /// Number of shards allowed to be destroyed before the program may close itself, if configured.
/// </summary> /// </summary>
private const int MaxDestroyedShards = 10; // TODO make configurable private const int MaxDestroyedShards = 10; // TODO make configurable
@ -26,7 +26,7 @@ class ShardManager : IDisposable {
/// Number of concurrent shard startups to happen on each check. /// Number of concurrent shard startups to happen on each check.
/// This value is also used in <see cref="DataRetention"/>. /// This value is also used in <see cref="DataRetention"/>.
/// </summary> /// </summary>
public const int MaxConcurrentOperations = 5; public const int MaxConcurrentOperations = 4;
/// <summary> /// <summary>
/// Amount of time without a completed background service run before a shard instance /// Amount of time without a completed background service run before a shard instance