Noi/BirthdayBot
1
0
Fork 0
mirror of https://github.com/NoiTheCat/BirthdayBot.git synced 2024-11-21 13:54:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Set up useful exit codes

Browse source 
Additionally, updated style on affected files and cleaned up certain parts of the code in each.
This commit is contained in:
Noi 2021-10-18 00:26:25 -07:00
parent 6f34fbe657
commit fdffa5425c
2 changed files with 249 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
Program.cs
View file

@ -2,57 +2,54 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace BirthdayBot namespace BirthdayBot;
{
class Program
{
private static ShardManager _bot;
public static DateTimeOffset BotStartTime { get; private set; }
static async Task Main() class Program {
{ private static ShardManager _bot;
BotStartTime = DateTimeOffset.UtcNow; public static DateTimeOffset BotStartTime { get; } = DateTimeOffset.UtcNow;
var cfg = new Configuration();
static async Task Main() {
var cfg = new Configuration();
try {
await Database.DoInitialDatabaseSetupAsync(); await Database.DoInitialDatabaseSetupAsync();
} catch (Npgsql.NpgsqlException e) {
Console.CancelKeyPress += OnCancelKeyPressed; Console.WriteLine("Error when attempting to connect to database: " + e.Message);
_bot = new ShardManager(cfg); Environment.Exit(1);
await Task.Delay(-1);
} }
Console.CancelKeyPress += OnCancelKeyPressed;
_bot = new ShardManager(cfg);
/// <summary> await Task.Delay(-1);
/// Sends a formatted message to console. }
/// </summary>
public static void Log(string source, string message) /// <summary>
{ /// Sends a formatted message to console.
var ts = DateTime.UtcNow; /// </summary>
var ls = new string[]{ "\r\n", "\n" }; public static void Log(string source, string message) {
foreach (var item in message.Split(ls, StringSplitOptions.None)) var ts = DateTime.UtcNow;
Console.WriteLine($"{ts:u} [{source}] {item}"); var ls = new string[] { "\r\n", "\n" };
} foreach (var item in message.Split(ls, StringSplitOptions.None))
Console.WriteLine($"{ts:u} [{source}] {item}");
private static void OnCancelKeyPressed(object sender, ConsoleCancelEventArgs e) }
{
e.Cancel = true; private static void OnCancelKeyPressed(object sender, ConsoleCancelEventArgs e) {
Log("Shutdown", "Captured cancel key; sending shutdown."); e.Cancel = true;
ProgramStop(); Log("Shutdown", "Captured cancel key; sending shutdown.");
} ProgramStop();
}
private static bool _stopping = false;
public static void ProgramStop() private static bool _stopping = false;
{ public static void ProgramStop() {
if (_stopping) return; if (_stopping) return;
_stopping = true; _stopping = true;
Log("Shutdown", "Commencing shutdown..."); Log("Shutdown", "Commencing shutdown...");
var dispose = Task.Run(_bot.Dispose); var dispose = Task.Run(_bot.Dispose);
if (!dispose.Wait(90000)) if (!dispose.Wait(90000)) {
{ Log("Shutdown", "Normal shutdown has not concluded after 90 seconds. Will force quit.");
Log("Shutdown", "Normal shutdown has not concluded after 90 seconds. Will force quit."); Environment.ExitCode += 0x200;
}
Environment.Exit(0);
} }
Environment.Exit(Environment.ExitCode);
} }
} }

432
ShardManager.cs
View file

@ -10,246 +10,226 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using static BirthdayBot.UserInterface.CommandsCommon; using static BirthdayBot.UserInterface.CommandsCommon;
namespace BirthdayBot namespace BirthdayBot;
{
/// <summary>
/// More or less the main class for the program. Handles individual shards and provides frequent
/// status reports regarding the overall health of the application.
/// </summary>
class ShardManager : IDisposable {
/// <summary> /// <summary>
/// The highest level part of this bot: /// Number of seconds between each time the status task runs, in seconds.
/// Starts up, looks over, and manages shard instances while containing common resources
/// and providing common functions for all existing shards.
/// </summary> /// </summary>
class ShardManager : IDisposable private const int StatusInterval = 90;
{
/// <summary>
/// Number of seconds between each time the manager's watchdog task runs, in seconds.
/// </summary>
private const int WatchdogInterval = 90;
/// <summary> /// <summary>
/// Number of shards allowed to be destroyed before forcing the program to close. /// Number of shards allowed to be destroyed before forcing the program to close.
/// </summary> /// </summary>
private const int MaxDestroyedShards = 10; // TODO make configurable private const int MaxDestroyedShards = 10; // TODO make configurable
/// <summary> /// <summary>
/// Number of concurrent shard startups to happen on each check. /// Number of concurrent shard startups to happen on each check.
/// This value is also used in <see cref="DataRetention"/>. /// This value is also used in <see cref="DataRetention"/>.
/// </summary> /// </summary>
public const int MaxConcurrentOperations = 5; public const int MaxConcurrentOperations = 5;
/// <summary> /// <summary>
/// Amount of time without a completed background service run before a shard instance /// Amount of time without a completed background service run before a shard instance
/// is considered "dead" and tasked to be removed. /// is considered "dead" and tasked to be removed. A fraction of this value is also used
/// </summary> /// to determine when a shard is "slow".
private static readonly TimeSpan DeadShardThreshold = new(0, 20, 0); /// </summary>
private static readonly TimeSpan DeadShardThreshold = new(0, 20, 0);
/// <summary> /// <summary>
/// A dictionary with shard IDs as its keys and shard instances as its values. /// A dictionary with shard IDs as its keys and shard instances as its values.
/// When initialized, all keys will be created as configured. If an instance is removed, /// When initialized, all keys will be created as configured. If an instance is removed,
/// a key's corresponding value will temporarily become null instead of the key/value /// a key's corresponding value will temporarily become null instead of the key/value
/// pair being removed. /// pair being removed.
/// </summary> /// </summary>
private readonly Dictionary<int, ShardInstance> _shards; private readonly Dictionary<int, ShardInstance> _shards;
// Commonly used command handler instances private readonly Dictionary<string, CommandHandler> _dispatchCommands;
private readonly Dictionary<string, CommandHandler> _dispatchCommands; private readonly UserCommands _cmdsUser;
private readonly UserCommands _cmdsUser; private readonly ListingCommands _cmdsListing;
private readonly ListingCommands _cmdsListing; private readonly HelpInfoCommands _cmdsHelp;
private readonly HelpInfoCommands _cmdsHelp; private readonly ManagerCommands _cmdsMods;
private readonly ManagerCommands _cmdsMods;
// Watchdog stuff private readonly Task _statusTask;
private readonly Task _watchdogTask; private readonly CancellationTokenSource _mainCancel;
private readonly CancellationTokenSource _watchdogCancel; private int _destroyedShards = 0;
private int _destroyedShards = 0;
internal Configuration Config { get; }
public ShardManager(Configuration cfg) internal Configuration Config { get; }
{
var ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
Log($"Birthday Bot v{ver.ToString(3)} is starting...");
Config = cfg; public ShardManager(Configuration cfg) {
var ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
Log($"Birthday Bot v{ver!.ToString(3)} is starting...");
// Command handler setup Config = cfg;
_dispatchCommands = new Dictionary<string, CommandHandler>(StringComparer.OrdinalIgnoreCase);
_cmdsUser = new UserCommands(cfg);
foreach (var item in _cmdsUser.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsListing = new ListingCommands(cfg);
foreach (var item in _cmdsListing.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsHelp = new HelpInfoCommands(cfg);
foreach (var item in _cmdsHelp.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsMods = new ManagerCommands(cfg, _cmdsUser.Commands);
foreach (var item in _cmdsMods.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_shards = new Dictionary<int, ShardInstance>(); // Command handler setup
// Create only the specified shards as needed by this instance _dispatchCommands = new Dictionary<string, CommandHandler>(StringComparer.OrdinalIgnoreCase);
for (int i = Config.ShardStart; i < (Config.ShardStart + Config.ShardAmount); i++) _cmdsUser = new UserCommands(cfg);
{ foreach (var item in _cmdsUser.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_shards.Add(i, null); _cmdsListing = new ListingCommands(cfg);
} foreach (var item in _cmdsListing.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsHelp = new HelpInfoCommands(cfg);
foreach (var item in _cmdsHelp.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsMods = new ManagerCommands(cfg, _cmdsUser.Commands);
foreach (var item in _cmdsMods.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
// Start watchdog // Allocate shards based on configuration
_watchdogCancel = new CancellationTokenSource(); _shards = new Dictionary<int, ShardInstance>();
_watchdogTask = Task.Factory.StartNew(WatchdogLoop, _watchdogCancel.Token, for (int i = Config.ShardStart; i < (Config.ShardStart + Config.ShardAmount); i++) {
TaskCreationOptions.LongRunning, TaskScheduler.Default); _shards.Add(i, null);
} }
public void Dispose() // Start status reporting thread
{ _mainCancel = new CancellationTokenSource();
_watchdogCancel.Cancel(); _statusTask = Task.Factory.StartNew(StatusLoop, _mainCancel.Token,
_watchdogTask.Wait(5000); TaskCreationOptions.LongRunning, TaskScheduler.Default);
if (!_watchdogTask.IsCompleted)
Log("Warning: Shard status watcher has not ended in time. Continuing...");
Log("Shutting down all shards...");
var shardDisposes = new List<Task>();
foreach (var item in _shards)
{
if (item.Value == null) continue;
shardDisposes.Add(Task.Run(item.Value.Dispose));
}
if (!Task.WhenAll(shardDisposes).Wait(60000))
{
Log("Warning: All shards did not properly stop after 60 seconds. Continuing...");
}
Log($"Shutdown complete. Bot uptime: {Common.BotUptime}");
}
private void Log(string message) => Program.Log(nameof(ShardManager), message);
/// <summary>
/// Creates and sets up a new shard instance.
/// </summary>
private async Task<ShardInstance> InitializeShard(int shardId)
{
ShardInstance newInstance;
var clientConf = new DiscordSocketConfig()
{
ShardId = shardId,
TotalShards = Config.ShardTotal,
LogLevel = LogSeverity.Info,
DefaultRetryMode = RetryMode.RetryRatelimit,
MessageCacheSize = 0, // not needed at all
GatewayIntents = GatewayIntents.Guilds | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages
};
var newClient = new DiscordSocketClient(clientConf);
newInstance = new ShardInstance(this, newClient, _dispatchCommands);
await newInstance.StartAsync().ConfigureAwait(false);
return newInstance;
}
private async Task WatchdogLoop()
{
try
{
while (!_watchdogCancel.IsCancellationRequested)
{
Log($"Bot uptime: {Common.BotUptime}");
// Iterate through shard list, extract data
var guildInfo = new Dictionary<int, (int, TimeSpan, string)>();
var now = DateTimeOffset.UtcNow;
var nullShards = new List<int>();
foreach (var item in _shards)
{
if (item.Value == null)
{
nullShards.Add(item.Key);
continue;
}
var shard = item.Value;
var guildCount = shard.DiscordClient.Guilds.Count;
var lastRun = now - shard.LastBackgroundRun;
var lastExec = shard.CurrentExecutingService ?? "null";
guildInfo[item.Key] = (guildCount, lastRun, lastExec);
}
// Process info
var guildCounts = guildInfo.Select(i => i.Value.Item1);
var guildTotal = guildCounts.Sum();
var guildAverage = guildCounts.Any() ? guildCounts.Average() : 0;
Log($"Currently in {guildTotal} guilds. Average shard load: {guildAverage:0.0}.");
// Health report
var goodShards = new List<int>();
var badShards = new List<int>(); // shards with low connection score OR long time since last work
var deadShards = new List<int>(); // shards to destroy and reinitialize
foreach (var item in guildInfo)
{
var lastRun = item.Value.Item2;
if (lastRun > DeadShardThreshold / 3)
{
badShards.Add(item.Key);
// Consider a shard dead after a long span without background activity
if (lastRun > DeadShardThreshold)
deadShards.Add(item.Key);
}
else
{
goodShards.Add(item.Key);
}
}
string statusDisplay(IEnumerable<int> list, bool detailedInfo)
{
if (!list.Any()) return "--";
var result = new StringBuilder();
foreach (var item in list)
{
result.Append(item.ToString("00") + " ");
if (detailedInfo)
{
result.Remove(result.Length - 1, 1);
result.Append($"[{Math.Floor(guildInfo[item].Item2.TotalSeconds):000}s");
result.Append($" {guildInfo[item].Item3}] ");
}
}
if (result.Length > 0) result.Remove(result.Length - 1, 1);
return result.ToString();
}
Log("Stable shards: " + statusDisplay(goodShards, false));
if (badShards.Count > 0) Log("Unstable shards: " + statusDisplay(badShards, true));
if (deadShards.Count > 0) Log("Shards to be restarted: " + statusDisplay(deadShards, false));
if (nullShards.Count > 0) Log("Inactive shards: " + statusDisplay(nullShards, false));
// Remove dead shards
foreach (var dead in deadShards) {
// TODO investigate - has this been hanging here?
_shards[dead].Dispose();
_shards[dead] = null;
_destroyedShards++;
}
if (Config.QuitOnFails && _destroyedShards > MaxDestroyedShards)
{
Program.ProgramStop();
}
else
{
// Start up any missing shards
int startAllowance = MaxConcurrentOperations;
foreach (var id in nullShards)
{
// To avoid possible issues with resources strained over so many shards starting at once,
// initialization is spread out by only starting a few at a time.
if (startAllowance-- > 0)
{
_shards[id] = await InitializeShard(id).ConfigureAwait(false);
}
else break;
}
}
// All done for now
await Task.Delay(WatchdogInterval * 1000, _watchdogCancel.Token).ConfigureAwait(false);
}
}
catch (TaskCanceledException) { }
}
} }
public void Dispose() {
_mainCancel.Cancel();
_statusTask.Wait(10000);
if (!_statusTask.IsCompleted)
Log("Warning: Main thread did not cleanly finish up in time. Continuing...");
Log("Shutting down all shards...");
var shardDisposes = new List<Task>();
foreach (var item in _shards) {
if (item.Value == null) continue;
shardDisposes.Add(Task.Run(item.Value.Dispose));
}
if (!Task.WhenAll(shardDisposes).Wait(30000)) {
Log("Warning: Not all shards terminated cleanly after 30 seconds. Continuing...");
}
Log($"Uptime: {Common.BotUptime}");
}
private void Log(string message) => Program.Log(nameof(ShardManager), message);
/// <summary>
/// Creates and sets up a new shard instance.
/// </summary>
private async Task<ShardInstance> InitializeShard(int shardId) {
ShardInstance newInstance;
var clientConf = new DiscordSocketConfig() {
ShardId = shardId,
TotalShards = Config.ShardTotal,
LogLevel = LogSeverity.Info,
DefaultRetryMode = RetryMode.RetryRatelimit,
MessageCacheSize = 0, // not needed at all
GatewayIntents = GatewayIntents.Guilds | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages
};
var newClient = new DiscordSocketClient(clientConf);
newInstance = new ShardInstance(this, newClient, _dispatchCommands);
await newInstance.StartAsync().ConfigureAwait(false);
return newInstance;
}
#region Status checking and display
private struct GuildStatusData {
public int GuildCount;
public TimeSpan LastTaskRunTime;
public string? ExecutingTask;
}
private string StatusDisplay(IEnumerable<int> guildList, Dictionary<int, GuildStatusData> guildInfo, bool showDetail) {
if (!guildList.Any()) return "--";
var result = new StringBuilder();
foreach (var item in guildList) {
result.Append(item.ToString("00") + " ");
if (showDetail) {
result.Remove(result.Length - 1, 1);
result.Append($"[{Math.Floor(guildInfo[item].LastTaskRunTime.TotalSeconds):000}s");
if (guildInfo[item].ExecutingTask != null)
result.Append($" {guildInfo[item].ExecutingTask}");
result.Append("] ");
}
}
if (result.Length > 0) result.Remove(result.Length - 1, 1);
return result.ToString();
}
private async Task StatusLoop() {
try {
while (!_mainCancel.IsCancellationRequested) {
Log($"Bot uptime: {Common.BotUptime}");
// Iterate through shard list, extract data
var guildInfo = new Dictionary<int, GuildStatusData>();
var now = DateTimeOffset.UtcNow;
var nullShards = new List<int>();
foreach (var item in _shards) {
if (item.Value == null) {
nullShards.Add(item.Key);
continue;
}
var shard = item.Value;
guildInfo[item.Key] = new GuildStatusData {
GuildCount = shard.DiscordClient.Guilds.Count,
LastTaskRunTime = now - shard.LastBackgroundRun,
ExecutingTask = shard.CurrentExecutingService
};
}
// Process info
var guildCounts = guildInfo.Select(i => i.Value.GuildCount);
var guildTotal = guildCounts.Sum();
var guildAverage = guildCounts.Any() ? guildCounts.Average() : 0;
Log($"Currently in {guildTotal} guilds. Average shard load: {guildAverage:0.0}.");
// Health report
var goodShards = new List<int>();
var badShards = new List<int>(); // shards with low connection score OR long time since last work
var deadShards = new List<int>(); // shards to destroy and reinitialize
foreach (var item in guildInfo) {
var lastRun = item.Value.LastTaskRunTime;
if (lastRun > DeadShardThreshold / 3) {
badShards.Add(item.Key);
// Consider a shard dead after a long span without background activity
if (lastRun > DeadShardThreshold)
deadShards.Add(item.Key);
} else {
goodShards.Add(item.Key);
}
}
Log("Online: " + StatusDisplay(goodShards, guildInfo, false));
if (badShards.Count > 0) Log("Slow: " + StatusDisplay(badShards, guildInfo, true));
if (deadShards.Count > 0) Log("Dead: " + StatusDisplay(deadShards, guildInfo, false));
if (nullShards.Count > 0) Log("Offline: " + StatusDisplay(nullShards, guildInfo, false));
// Remove dead shards
foreach (var dead in deadShards) {
_shards[dead].Dispose();
_shards[dead] = null;
_destroyedShards++;
}
if (Config.QuitOnFails && _destroyedShards > MaxDestroyedShards) {
Environment.ExitCode = 0x04;
Program.ProgramStop();
} else {
// Start up any missing shards
int startAllowance = MaxConcurrentOperations;
foreach (var id in nullShards) {
// To avoid possible issues with resources strained over so many shards starting at once,
// initialization is spread out by only starting a few at a time.
if (startAllowance-- > 0) {
_shards[id] = await InitializeShard(id).ConfigureAwait(false);
} else break;
}
}
await Task.Delay(StatusInterval * 1000, _mainCancel.Token).ConfigureAwait(false);
}
} catch (TaskCanceledException) { }
}
#endregion
} }