mirror of
https://github.com/NoiTheCat/BirthdayBot.git
synced 2024-11-25 01:44:12 +00:00
Increase threshold, no longer stagger DataRetention
This staggering was most likely the cause of a major amount of data lost recently, in which certain guilds never had their values updated. The staggering was meant to attempt to reduce load on a server with limited capabilities, and testing shows that it runs on more capable hardware without this issue when run this way.
This commit is contained in:
parent
67f78e068e
commit
f8350fed53
3 changed files with 19 additions and 19 deletions
|
@ -1,11 +1,6 @@
|
||||||
using BirthdayBot.Data;
|
using BirthdayBot.Data;
|
||||||
using NpgsqlTypes;
|
using NpgsqlTypes;
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace BirthdayBot.BackgroundServices;
|
namespace BirthdayBot.BackgroundServices;
|
||||||
|
|
||||||
|
@ -14,14 +9,16 @@ namespace BirthdayBot.BackgroundServices;
|
||||||
/// </summary>
|
/// </summary>
|
||||||
class DataRetention : BackgroundService {
|
class DataRetention : BackgroundService {
|
||||||
private static readonly SemaphoreSlim _updateLock = new(ShardManager.MaxConcurrentOperations);
|
private static readonly SemaphoreSlim _updateLock = new(ShardManager.MaxConcurrentOperations);
|
||||||
const int ProcessInterval = 3600 / ShardBackgroundWorker.Interval; // Process about once per hour
|
const int ProcessInterval = 5400 / ShardBackgroundWorker.Interval; // Process about once per hour and a half
|
||||||
const int Stagger = 3; // How many ticks in between each group of guilds to stagger processing.
|
// Amount of days without updates before data is considered stale and up for deletion.
|
||||||
|
const int StaleGuildThreshold = 180;
|
||||||
|
const int StaleUserThreashold = 360;
|
||||||
|
|
||||||
public DataRetention(ShardInstance instance) : base(instance) { }
|
public DataRetention(ShardInstance instance) : base(instance) { }
|
||||||
|
|
||||||
public override async Task OnTick(int tickCount, CancellationToken token) {
|
public override async Task OnTick(int tickCount, CancellationToken token) {
|
||||||
// On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks.
|
// On each tick, run only a set group of guilds, each group still processed every ProcessInterval ticks.
|
||||||
if ((tickCount + ShardInstance.ShardId * Stagger) % ProcessInterval != 0) return;
|
if ((tickCount + ShardInstance.ShardId) % ProcessInterval != 0) return;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// A semaphore is used to restrict this work being done concurrently on other shards
|
// A semaphore is used to restrict this work being done concurrently on other shards
|
||||||
|
@ -33,7 +30,7 @@ class DataRetention : BackgroundService {
|
||||||
throw new TaskCanceledException();
|
throw new TaskCanceledException();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
// Build a list of all values to update
|
// Build a list of all values across all guilds to update
|
||||||
var updateList = new Dictionary<ulong, List<ulong>>();
|
var updateList = new Dictionary<ulong, List<ulong>>();
|
||||||
foreach (var g in ShardInstance.DiscordClient.Guilds) {
|
foreach (var g in ShardInstance.DiscordClient.Guilds) {
|
||||||
// Get list of IDs for all users who exist in the database and currently exist in the guild
|
// Get list of IDs for all users who exist in the database and currently exist in the guild
|
||||||
|
@ -80,17 +77,18 @@ class DataRetention : BackgroundService {
|
||||||
var resultText = new StringBuilder();
|
var resultText = new StringBuilder();
|
||||||
resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users.");
|
resultText.Append($"Updated {updatedGuilds} guilds, {updatedUsers} users.");
|
||||||
|
|
||||||
// Delete all old values - expects referencing tables to have 'on delete cascade'
|
// Deletes both guild and user data if it hasn't been seen for over the threshold defined at the top of this file
|
||||||
|
// Expects referencing tables to have 'on delete cascade'
|
||||||
using var t = db.BeginTransaction();
|
using var t = db.BeginTransaction();
|
||||||
int staleGuilds, staleUsers;
|
int staleGuilds, staleUsers;
|
||||||
using (var c = db.CreateCommand()) {
|
using (var c = db.CreateCommand()) {
|
||||||
// Delete data for guilds not seen in 4 weeks
|
c.CommandText = $"delete from {GuildConfiguration.BackingTable}" +
|
||||||
c.CommandText = $"delete from {GuildConfiguration.BackingTable} where (now() - interval '28 days') > last_seen";
|
$" where (now() - interval '{StaleGuildThreshold} days') > last_seen";
|
||||||
staleGuilds = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
staleGuilds = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
using (var c = db.CreateCommand()) {
|
using (var c = db.CreateCommand()) {
|
||||||
// Delete data for users not seen in 8 weeks
|
c.CommandText = $"delete from {GuildUserConfiguration.BackingTable}" +
|
||||||
c.CommandText = $"delete from {GuildUserConfiguration.BackingTable} where (now() - interval '56 days') > last_seen";
|
$" where (now() - interval '{StaleUserThreashold} days') > last_seen";
|
||||||
staleUsers = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
staleUsers = await c.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
|
||||||
}
|
}
|
||||||
if (staleGuilds != 0 || staleUsers != 0) {
|
if (staleGuilds != 0 || staleUsers != 0) {
|
||||||
|
@ -100,7 +98,7 @@ class DataRetention : BackgroundService {
|
||||||
if (staleUsers != 0) resultText.Append(", ");
|
if (staleUsers != 0) resultText.Append(", ");
|
||||||
}
|
}
|
||||||
if (staleUsers != 0) {
|
if (staleUsers != 0) {
|
||||||
resultText.Append($"{staleUsers} standalone users");
|
resultText.Append($"{staleUsers} users");
|
||||||
}
|
}
|
||||||
resultText.Append('.');
|
resultText.Append('.');
|
||||||
}
|
}
|
||||||
|
|
|
@ -80,12 +80,14 @@ class ShardInstance : IDisposable {
|
||||||
|
|
||||||
public void Log(string source, string message) => Program.Log($"Shard {ShardId:00}] [{source}", message);
|
public void Log(string source, string message) => Program.Log($"Shard {ShardId:00}] [{source}", message);
|
||||||
|
|
||||||
|
public void RequestDownloadUsers(ulong guildId) => _background.UserDownloader.RequestDownload(guildId);
|
||||||
|
|
||||||
#region Event handling
|
#region Event handling
|
||||||
private Task Client_Log(LogMessage arg) {
|
private Task Client_Log(LogMessage arg) {
|
||||||
// TODO revise this some time, filters might need to be modified by now
|
// TODO revise this some time, filters might need to be modified by now
|
||||||
// Suppress certain messages
|
// Suppress certain messages
|
||||||
if (arg.Message != null) {
|
if (arg.Message != null) {
|
||||||
if (arg.Message.StartsWith("Unknown Dispatch ") || arg.Message.StartsWith("Missing Channel")) return Task.CompletedTask;
|
if (arg.Message.StartsWith("Unknown Dispatch ") || arg.Message.StartsWith("Unknown Channel")) return Task.CompletedTask;
|
||||||
switch (arg.Message) // Connection status messages replaced by ShardManager's output
|
switch (arg.Message) // Connection status messages replaced by ShardManager's output
|
||||||
{
|
{
|
||||||
case "Connecting":
|
case "Connecting":
|
||||||
|
|
|
@ -15,10 +15,10 @@ class ShardManager : IDisposable {
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Number of seconds between each time the status task runs, in seconds.
|
/// Number of seconds between each time the status task runs, in seconds.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private const int StatusInterval = 90;
|
private const int StatusInterval = 60;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Number of shards allowed to be destroyed before forcing the program to close.
|
/// Number of shards allowed to be destroyed before the program may close itself, if configured.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private const int MaxDestroyedShards = 10; // TODO make configurable
|
private const int MaxDestroyedShards = 10; // TODO make configurable
|
||||||
|
|
||||||
|
@ -26,7 +26,7 @@ class ShardManager : IDisposable {
|
||||||
/// Number of concurrent shard startups to happen on each check.
|
/// Number of concurrent shard startups to happen on each check.
|
||||||
/// This value is also used in <see cref="DataRetention"/>.
|
/// This value is also used in <see cref="DataRetention"/>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public const int MaxConcurrentOperations = 5;
|
public const int MaxConcurrentOperations = 4;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Amount of time without a completed background service run before a shard instance
|
/// Amount of time without a completed background service run before a shard instance
|
||||||
|
|
Loading…
Reference in a new issue