Noi/BirthdayBot
1
0
Fork 0
mirror of https://github.com/NoiTheCat/BirthdayBot.git synced 2024-10-16 15:59:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Rework member cache downloading

Browse source 
Fill it quicker when immediately needed, removing the need for a request list and all relating to it in the corresponding background service.

Additionally, updated usings, style, nullables in all affected files.
This commit is contained in:
Noi 2021-11-22 13:40:08 -08:00
parent 488ebfa163
commit 5e4d030467
8 changed files with 373 additions and 487 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
BackgroundServices/BirthdayRoleUpdate.cs
View file

@ -1,12 +1,6 @@
using BirthdayBot.Data;
using Discord.WebSocket;
using NodaTime;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace BirthdayBot.BackgroundServices;
@ -46,7 +40,7 @@ class BirthdayRoleUpdate : BackgroundService {
/// </summary>
private static async Task ProcessGuildAsync(SocketGuild guild) {
// Load guild information - stop if local cache is unavailable.
if (!Common.HasMostMembersDownloaded(guild)) return;
if (!guild.HasAllMembers) return;
var gc = await GuildConfiguration.LoadAsync(guild.Id, true).ConfigureAwait(false);
if (gc == null) return;

30
BackgroundServices/SelectiveAutoUserDownload.cs
View file

@ -1,40 +1,22 @@
using BirthdayBot.Data;
using Discord;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace BirthdayBot.BackgroundServices;
/// <summary>
/// Rather than use up unnecessary resources by auto-downloading the user list in -every-
/// server we're in, this service checks if fetching the user list is warranted for each
/// guild before proceeding to request it.
/// Proactively fills the user cache for guilds in which any birthday data already exists.
/// </summary>
class SelectiveAutoUserDownload : BackgroundService {
private readonly HashSet<ulong> _fetchRequests = new();
public SelectiveAutoUserDownload(ShardInstance instance) : base(instance) { }
public override async Task OnTick(int tickCount, CancellationToken token) {
IEnumerable<ulong> requests;
lock (_fetchRequests) {
requests = _fetchRequests.ToArray();
_fetchRequests.Clear();
}
foreach (var guild in ShardInstance.DiscordClient.Guilds) {
// Has the potential to disconnect while in the middle of processing.
if (ShardInstance.DiscordClient.ConnectionState != ConnectionState.Connected) return;
// Determine if there is action to be taken...
if (guild.HasAllMembers) continue;
if (requests.Contains(guild.Id) || await GuildUserAnyAsync(guild.Id).ConfigureAwait(false)) {
await guild.DownloadUsersAsync().ConfigureAwait(false);
// Must delay after a download request. Seems to hang indefinitely otherwise.
await Task.Delay(300, CancellationToken.None).ConfigureAwait(false);
if (!guild.HasAllMembers && await GuildUserAnyAsync(guild.Id).ConfigureAwait(false)) {
await guild.DownloadUsersAsync().ConfigureAwait(false); // This is already on a separate thread; no need to Task.Run
await Task.Delay(200, CancellationToken.None).ConfigureAwait(false); // Must delay, or else it seems to hang...
}
}
}
@ -52,8 +34,4 @@ class SelectiveAutoUserDownload : BackgroundService {
using var r = await c.ExecuteReaderAsync(CancellationToken.None).ConfigureAwait(false);
return r.Read();
}
public void RequestDownload(ulong guildId) {
lock (_fetchRequests) _fetchRequests.Add(guildId);
}
}

2
BirthdayBot.csproj
View file

@ -5,7 +5,7 @@
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Version>3.2.2</Version>
<Version>3.2.3</Version>
<Authors>NoiTheCat</Authors>
</PropertyGroup>

21
Common.cs
View file

@ -1,6 +1,4 @@
using Discord.WebSocket;
using System.Collections.Generic;
using System.Text;
using System.Text;
namespace BirthdayBot;
@ -33,21 +31,4 @@ static class Common {
{ 1, "Jan" }, { 2, "Feb" }, { 3, "Mar" }, { 4, "Apr" }, { 5, "May" }, { 6, "Jun" },
{ 7, "Jul" }, { 8, "Aug" }, { 9, "Sep" }, { 10, "Oct" }, { 11, "Nov" }, { 12, "Dec" }
};
/// <summary>
/// An alternative to <see cref="SocketGuild.HasAllMembers"/>.
/// Returns true if *most* members have been downloaded.
/// </summary>
public static bool HasMostMembersDownloaded(SocketGuild guild) {
if (guild.HasAllMembers) return true;
if (guild.MemberCount > 30) {
// For guilds of size over 30, require 85% or more of the members to be known
// (26/30, 42/50, 255/300, etc)
int threshold = (int)(guild.MemberCount * 0.85);
return guild.DownloadedMemberCount >= threshold;
} else {
// For smaller guilds, fail if two or more members are missing
return guild.MemberCount - guild.DownloadedMemberCount <= 2;
}
}
}

2
ShardInstance.cs
View file

@ -80,8 +80,6 @@ class ShardInstance : IDisposable {
public void Log(string source, string message) => Program.Log($"Shard {ShardId:00}] [{source}", message);
public void RequestDownloadUsers(ulong guildId) => _background.UserDownloader.RequestDownload(guildId);
#region Event handling
private Task Client_Log(LogMessage arg) {
// TODO revise this some time, filters might need to be modified by now

73
UserInterface/CommandsCommon.cs
View file

@ -1,18 +1,14 @@
using BirthdayBot.Data;
using Discord.WebSocket;
using NodaTime;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace BirthdayBot.UserInterface
{
namespace BirthdayBot.UserInterface;
/// <summary>
/// Common base class for common constants and variables.
/// </summary>
internal abstract class CommandsCommon
{
internal abstract class CommandsCommon {
#if DEBUG
public const string CommandPrefix = "bt.";
#else
@ -22,35 +18,27 @@ namespace BirthdayBot.UserInterface
public const string ParameterError = ":x: Invalid usage. Refer to how to use the command and try again.";
public const string NoParameterError = ":x: This command does not accept any parameters.";
public const string InternalError = ":x: An internal bot error occurred. The bot maintainer has been notified of the issue.";
public const string UsersNotDownloadedError = ":eight_spoked_asterisk: Still catching up... Please try the command again in a few minutes.";
public const string MemberCacheEmptyError = ":warning: Please try the command again.";
public delegate Task CommandHandler(ShardInstance instance, GuildConfiguration gconf,
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser);
protected static Dictionary<string, string> TzNameMap {
get {
// Because IDateTimeZoneProvider.GetZoneOrNull is not case sensitive:
// Getting every existing zone name and mapping it onto a dictionary. Now a case-insensitive
// search can be made with the accepted value retrieved as a result.
if (_tzNameMap == null)
{
_tzNameMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var name in DateTimeZoneProviders.Tzdb.Ids) _tzNameMap.Add(name, name);
}
return _tzNameMap;
}
}
protected static ReadOnlyDictionary<string, string> TzNameMap { get; }
protected static Regex ChannelMention { get; } = new Regex(@"<#(\d+)>");
protected static Regex UserMention { get; } = new Regex(@"\!?(\d+)>");
private static Dictionary<string, string> _tzNameMap; // Value set by getter property on first read
protected Configuration BotConfig { get; }
protected CommandsCommon(Configuration db)
{
protected CommandsCommon(Configuration db) {
BotConfig = db;
}
static CommandsCommon() {
var dict = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var name in DateTimeZoneProviders.Tzdb.Ids) dict.Add(name, name);
TzNameMap = new(dict);
}
/// <summary>
/// On command dispatcher initialization, it will retrieve all available commands through here.
/// </summary>
@ -59,20 +47,10 @@ namespace BirthdayBot.UserInterface
/// <summary>
/// Checks given time zone input. Returns a valid string for use with NodaTime.
/// </summary>
protected static string ParseTimeZone(string tzinput)
{
string tz = null;
if (tzinput != null)
{
protected static string ParseTimeZone(string tzinput) {
if (tzinput.Equals("Asia/Calcutta", StringComparison.OrdinalIgnoreCase)) tzinput = "Asia/Kolkata";
// Just check if the input exists in the map. Get the "true" value, or reject it altogether.
if (!TzNameMap.TryGetValue(tzinput, out tz))
{
throw new FormatException(":x: Unexpected time zone name."
if (!TzNameMap.TryGetValue(tzinput, out string? tz)) throw new FormatException(":x: Unexpected time zone name."
+ $" Refer to `{CommandPrefix}help-tzdata` to help determine the correct value.");
}
}
return tz;
}
@ -80,8 +58,7 @@ namespace BirthdayBot.UserInterface
/// Given user input where a user-like parameter is expected, attempts to resolve to an ID value.
/// Input must be a mention or explicit ID. No name resolution is done here.
/// </summary>
protected static bool TryGetUserId(string input, out ulong result)
{
protected static bool TryGetUserId(string input, out ulong result) {
string doParse;
var m = UserMention.Match(input);
if (m.Success) doParse = m.Groups[1].Value;
@ -95,5 +72,23 @@ namespace BirthdayBot.UserInterface
result = default;
return false;
}
/// <summary>
/// An alternative to <see cref="SocketGuild.HasAllMembers"/> to be called by command handlers needing a full member cache.
/// Creates a download request if necessary.
/// </summary>
/// <returns>
/// True if the member cache is already filled, false otherwise.
/// </returns>
/// <remarks>
/// Any updates to the member cache aren't accessible until the event handler finishes execution, meaning proactive downloading
/// is necessary, and is handled by <seealso cref="BackgroundServices.SelectiveAutoUserDownload"/>. In situations where
/// this approach fails, this is to be called, and the user must be asked to attempt the command again if this returns false.
/// </remarks>
protected static async Task<bool> HasMemberCacheAsync(SocketGuild guild) {
if (guild.HasAllMembers) return true;
// Event handling thread hangs if awaited normally or used with Task.Run
await Task.Factory.StartNew(guild.DownloadUsersAsync).ConfigureAwait(false);
return false;
}
}

142
UserInterface/ListingCommands.cs
View file

@ -1,19 +1,12 @@
using BirthdayBot.Data;
using Discord.WebSocket;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace BirthdayBot.UserInterface
{
namespace BirthdayBot.UserInterface;
/// <summary>
/// Commands for listing upcoming and all birthdays.
/// </summary>
internal class ListingCommands : CommandsCommon
{
internal class ListingCommands : CommandsCommon {
public ListingCommands(Configuration db) : base(db) { }
public override IEnumerable<(string, CommandHandler)> Commands
@ -36,56 +29,45 @@ namespace BirthdayBot.UserInterface
#endregion
private async Task CmdWhen(ShardInstance instance, GuildConfiguration gconf,
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser)
{
if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
{
instance.RequestDownloadUsers(reqChannel.Guild.Id);
await reqChannel.SendMessageAsync(UsersNotDownloadedError);
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser) {
if (!await HasMemberCacheAsync(reqChannel.Guild)) {
await reqChannel.SendMessageAsync(MemberCacheEmptyError);
return;
}
// Requires a parameter
if (param.Length == 1)
{
if (param.Length == 1) {
await reqChannel.SendMessageAsync(ParameterError, embed: DocWhen.UsageEmbed).ConfigureAwait(false);
return;
}
var search = param[1];
if (param.Length == 3)
{
if (param.Length == 3) {
// param maxes out at 3 values. param[2] might contain part of the search string (if name has a space)
search += " " + param[2];
}
SocketGuildUser searchTarget = null;
SocketGuildUser? searchTarget = null;
if (!TryGetUserId(search, out ulong searchId)) // ID lookup
{
// name lookup without discriminator
foreach (var searchuser in reqChannel.Guild.Users)
{
if (string.Equals(search, searchuser.Username, StringComparison.OrdinalIgnoreCase))
{
foreach (var searchuser in reqChannel.Guild.Users) {
if (string.Equals(search, searchuser.Username, StringComparison.OrdinalIgnoreCase)) {
searchTarget = searchuser;
break;
}
}
}
else
{
} else {
searchTarget = reqChannel.Guild.GetUser(searchId);
}
if (searchTarget == null)
{
if (searchTarget == null) {
await reqChannel.SendMessageAsync(BadUserError, embed: DocWhen.UsageEmbed).ConfigureAwait(false);
return;
}
var searchTargetData = await GuildUserConfiguration.LoadAsync(reqChannel.Guild.Id, searchId).ConfigureAwait(false);
if (!searchTargetData.IsKnown)
{
if (!searchTargetData.IsKnown) {
await reqChannel.SendMessageAsync("I do not have birthday information for that user.").ConfigureAwait(false);
return;
}
@ -100,37 +82,29 @@ namespace BirthdayBot.UserInterface
// Creates a file with all birthdays.
private async Task CmdList(ShardInstance instance, GuildConfiguration gconf,
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser)
{
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser) {
// For now, we're restricting this command to moderators only. This may turn into an option later.
if (!gconf.IsBotModerator(reqUser))
{
if (!gconf.IsBotModerator(reqUser)) {
// Do not add detailed usage information to this error message.
await reqChannel.SendMessageAsync(":x: Only bot moderators may use this command.").ConfigureAwait(false);
return;
}
if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
{
instance.RequestDownloadUsers(reqChannel.Guild.Id);
await reqChannel.SendMessageAsync(UsersNotDownloadedError);
if (!await HasMemberCacheAsync(reqChannel.Guild)) {
await reqChannel.SendMessageAsync(MemberCacheEmptyError);
return;
}
bool useCsv = false;
// Check for CSV option
if (param.Length == 2)
{
if (param.Length == 2) {
if (param[1].ToLower() == "csv") useCsv = true;
else
{
else {
await reqChannel.SendMessageAsync(":x: That is not available as an export format.", embed: DocList.UsageEmbed)
.ConfigureAwait(false);
return;
}
}
else if (param.Length > 2)
{
} else if (param.Length > 2) {
await reqChannel.SendMessageAsync(ParameterError, embed: DocList.UsageEmbed).ConfigureAwait(false);
return;
}
@ -139,34 +113,24 @@ namespace BirthdayBot.UserInterface
var filepath = Path.GetTempPath() + "birthdaybot-" + reqChannel.Guild.Id;
string fileoutput;
if (useCsv)
{
if (useCsv) {
fileoutput = ListExportCsv(reqChannel, bdlist);
filepath += ".csv";
}
else
{
} else {
fileoutput = ListExportNormal(reqChannel, bdlist);
filepath += ".txt.";
}
await File.WriteAllTextAsync(filepath, fileoutput, Encoding.UTF8).ConfigureAwait(false);
try
{
try {
await reqChannel.SendFileAsync(filepath, $"Exported {bdlist.Count} birthdays to file.").ConfigureAwait(false);
}
catch (Discord.Net.HttpException)
{
} catch (Discord.Net.HttpException) {
reqChannel.SendMessageAsync(":x: Unable to send list due to a permissions issue. Check the 'Attach Files' permission.").Wait();
}
catch (Exception ex)
{
} catch (Exception ex) {
Program.Log("Listing", ex.ToString());
reqChannel.SendMessageAsync(InternalError).Wait();
// TODO webhook report
}
finally
{
} finally {
File.Delete(filepath);
}
}
@ -174,12 +138,9 @@ namespace BirthdayBot.UserInterface
// "Recent and upcoming birthdays"
// The 'recent' bit removes time zone ambiguity and spares us from extra time zone processing here
private async Task CmdUpcoming(ShardInstance instance, GuildConfiguration gconf,
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser)
{
if (!Common.HasMostMembersDownloaded(reqChannel.Guild))
{
instance.RequestDownloadUsers(reqChannel.Guild.Id);
await reqChannel.SendMessageAsync(UsersNotDownloadedError);
string[] param, SocketTextChannel reqChannel, SocketGuildUser reqUser) {
if (!await HasMemberCacheAsync(reqChannel.Guild)) {
await reqChannel.SendMessageAsync(MemberCacheEmptyError);
return;
}
@ -207,8 +168,7 @@ namespace BirthdayBot.UserInterface
// Build sorted name list
var names = new List<string>();
foreach (var item in results)
{
foreach (var item in results) {
names.Add(item.DisplayName);
}
names.Sort(StringComparer.OrdinalIgnoreCase);
@ -216,11 +176,9 @@ namespace BirthdayBot.UserInterface
var first = true;
output.AppendLine();
output.Append($"● `{Common.MonthNames[results.First().BirthMonth]}-{results.First().BirthDay:00}`: ");
foreach (var item in names)
{
foreach (var item in names) {
// If the output is starting to fill up, send out this message and prepare a new one.
if (output.Length > 800)
{
if (output.Length > 800) {
await reqChannel.SendMessageAsync(output.ToString()).ConfigureAwait(false);
output.Clear();
first = true;
@ -245,8 +203,7 @@ namespace BirthdayBot.UserInterface
/// Fetches all guild birthdays and places them into an easily usable structure.
/// Users currently not in the guild are not included in the result.
/// </summary>
private static async Task<List<ListItem>> GetSortedUsersAsync(SocketGuild guild)
{
private static async Task<List<ListItem>> GetSortedUsersAsync(SocketGuild guild) {
using var db = await Database.OpenConnectionAsync();
using var c = db.CreateCommand();
c.CommandText = "select user_id, birth_month, birth_day from " + GuildUserConfiguration.BackingTable
@ -255,8 +212,7 @@ namespace BirthdayBot.UserInterface
c.Prepare();
using var r = await c.ExecuteReaderAsync();
var result = new List<ListItem>();
while (await r.ReadAsync())
{
while (await r.ReadAsync()) {
var id = (ulong)r.GetInt64(0);
var month = r.GetInt32(1);
var day = r.GetInt32(2);
@ -264,8 +220,7 @@ namespace BirthdayBot.UserInterface
var guildUser = guild.GetUser(id);
if (guildUser == null) continue; // Skip user not in guild
result.Add(new ListItem()
{
result.Add(new ListItem() {
BirthMonth = month,
BirthDay = day,
DateIndex = DateIndex(month, day),
@ -276,14 +231,12 @@ namespace BirthdayBot.UserInterface
return result;
}
private string ListExportNormal(SocketGuildChannel channel, IEnumerable<ListItem> list)
{
private string ListExportNormal(SocketGuildChannel channel, IEnumerable<ListItem> list) {
// Output: "● Mon-dd: (user ID) Username [ - Nickname: (nickname)]"
var result = new StringBuilder();
result.AppendLine("Birthdays in " + channel.Guild.Name);
result.AppendLine();
foreach (var item in list)
{
foreach (var item in list) {
var user = channel.Guild.GetUser(item.UserId);
if (user == null) continue; // User disappeared in the instant between getting list and processing
result.Append($"● {Common.MonthNames[item.BirthMonth]}-{item.BirthDay:00}: ");
@ -295,16 +248,14 @@ namespace BirthdayBot.UserInterface
return result.ToString();
}
private string ListExportCsv(SocketGuildChannel channel, IEnumerable<ListItem> list)
{
private string ListExportCsv(SocketGuildChannel channel, IEnumerable<ListItem> list) {
// Output: User ID, Username, Nickname, Month-Day, Month, Day
var result = new StringBuilder();
// Conforming to RFC 4180; with header
result.Append("UserId,Username,Nickname,MonthDayDisp,Month,Day");
result.Append("\r\n"); // crlf line break is specified by the standard
foreach (var item in list)
{
foreach (var item in list) {
var user = channel.Guild.GetUser(item.UserId);
if (user == null) continue; // User disappeared in the instant between getting list and processing
result.Append(item.UserId);
@ -323,12 +274,10 @@ namespace BirthdayBot.UserInterface
return result.ToString();
}
private static string CsvEscape(string input)
{
private static string CsvEscape(string input) {
var result = new StringBuilder();
result.Append('"');
foreach (var ch in input)
{
foreach (var ch in input) {
if (ch == '"') result.Append('"');
result.Append(ch);
}
@ -336,8 +285,7 @@ namespace BirthdayBot.UserInterface
return result.ToString();
}
private static int DateIndex(int month, int day)
{
private static int DateIndex(int month, int day) {
var dateindex = 0;
// Add month offsets
if (month > 1) dateindex += 31; // Offset January
@ -355,8 +303,7 @@ namespace BirthdayBot.UserInterface
return dateindex;
}
private struct ListItem
{
private struct ListItem {
public int DateIndex;
public int BirthMonth;
public int BirthDay;
@ -364,4 +311,3 @@ namespace BirthdayBot.UserInterface
public string DisplayName;
}
}
}

12
UserInterface/ManagerCommands.cs
View file

@ -1,11 +1,6 @@
using BirthdayBot.Data;
using Discord.WebSocket;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace BirthdayBot.UserInterface;
@ -327,9 +322,8 @@ internal class ManagerCommands : CommandsCommon {
// Moderators only. As with config, silently drop if this check fails.
if (!gconf.IsBotModerator(reqUser)) return;
if (!Common.HasMostMembersDownloaded(reqChannel.Guild)) {
instance.RequestDownloadUsers(reqChannel.Guild.Id);
await reqChannel.SendMessageAsync(UsersNotDownloadedError);
if (!await HasMemberCacheAsync(reqChannel.Guild)) {
await reqChannel.SendMessageAsync(MemberCacheEmptyError);
return;
}
@ -393,7 +387,7 @@ internal class ManagerCommands : CommandsCommon {
var conf = await GuildConfiguration.LoadAsync(guild.Id, true).ConfigureAwait(false);
result.AppendLine($"Server ID: {guild.Id} | Bot shard ID: {instance.ShardId:00}");
bool hasMembers = Common.HasMostMembersDownloaded(guild);
bool hasMembers = guild.HasAllMembers;
result.Append(DoTestFor("Bot has obtained the user list", () => hasMembers));
result.AppendLine($" - Has {guild.DownloadedMemberCount} of {guild.MemberCount} members.");
int bdayCount = -1;