Noi/BirthdayBot
1
0
Fork 0
mirror of https://github.com/NoiTheCat/BirthdayBot.git synced 2024-10-16 07:59:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Set up useful exit codes

Browse source 
Additionally, updated style on affected files and cleaned up certain parts of the code in each.
This commit is contained in:
Noi 2021-10-18 00:26:25 -07:00
parent 6f34fbe657
commit fdffa5425c
2 changed files with 249 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

89
Program.cs
View file

@ -2,57 +2,54 @@
using System;
using System.Threading.Tasks;
namespace BirthdayBot
{
class Program
{
private static ShardManager _bot;
public static DateTimeOffset BotStartTime { get; private set; }
namespace BirthdayBot;
static async Task Main()
{
BotStartTime = DateTimeOffset.UtcNow;
var cfg = new Configuration();
class Program {
private static ShardManager _bot;
public static DateTimeOffset BotStartTime { get; } = DateTimeOffset.UtcNow;
static async Task Main() {
var cfg = new Configuration();
try {
await Database.DoInitialDatabaseSetupAsync();
Console.CancelKeyPress += OnCancelKeyPressed;
_bot = new ShardManager(cfg);
await Task.Delay(-1);
} catch (Npgsql.NpgsqlException e) {
Console.WriteLine("Error when attempting to connect to database: " + e.Message);
Environment.Exit(1);
}
Console.CancelKeyPress += OnCancelKeyPressed;
_bot = new ShardManager(cfg);
/// <summary>
/// Sends a formatted message to console.
/// </summary>
public static void Log(string source, string message)
{
var ts = DateTime.UtcNow;
var ls = new string[]{ "\r\n", "\n" };
foreach (var item in message.Split(ls, StringSplitOptions.None))
Console.WriteLine($"{ts:u} [{source}] {item}");
}
private static void OnCancelKeyPressed(object sender, ConsoleCancelEventArgs e)
{
e.Cancel = true;
Log("Shutdown", "Captured cancel key; sending shutdown.");
ProgramStop();
}
private static bool _stopping = false;
public static void ProgramStop()
{
if (_stopping) return;
_stopping = true;
Log("Shutdown", "Commencing shutdown...");
var dispose = Task.Run(_bot.Dispose);
if (!dispose.Wait(90000))
{
Log("Shutdown", "Normal shutdown has not concluded after 90 seconds. Will force quit.");
}
Environment.Exit(0);
await Task.Delay(-1);
}
/// <summary>
/// Sends a formatted message to console.
/// </summary>
public static void Log(string source, string message) {
var ts = DateTime.UtcNow;
var ls = new string[] { "\r\n", "\n" };
foreach (var item in message.Split(ls, StringSplitOptions.None))
Console.WriteLine($"{ts:u} [{source}] {item}");
}
private static void OnCancelKeyPressed(object sender, ConsoleCancelEventArgs e) {
e.Cancel = true;
Log("Shutdown", "Captured cancel key; sending shutdown.");
ProgramStop();
}
private static bool _stopping = false;
public static void ProgramStop() {
if (_stopping) return;
_stopping = true;
Log("Shutdown", "Commencing shutdown...");
var dispose = Task.Run(_bot.Dispose);
if (!dispose.Wait(90000)) {
Log("Shutdown", "Normal shutdown has not concluded after 90 seconds. Will force quit.");
Environment.ExitCode += 0x200;
}
Environment.Exit(Environment.ExitCode);
}
}

432
ShardManager.cs
View file

@ -10,246 +10,226 @@ using System.Threading;
using System.Threading.Tasks;
using static BirthdayBot.UserInterface.CommandsCommon;
namespace BirthdayBot
{
namespace BirthdayBot;
/// <summary>
/// More or less the main class for the program. Handles individual shards and provides frequent
/// status reports regarding the overall health of the application.
/// </summary>
class ShardManager : IDisposable {
/// <summary>
/// The highest level part of this bot:
/// Starts up, looks over, and manages shard instances while containing common resources
/// and providing common functions for all existing shards.
/// Number of seconds between each time the status task runs, in seconds.
/// </summary>
class ShardManager : IDisposable
{
/// <summary>
/// Number of seconds between each time the manager's watchdog task runs, in seconds.
/// </summary>
private const int WatchdogInterval = 90;
private const int StatusInterval = 90;
/// <summary>
/// Number of shards allowed to be destroyed before forcing the program to close.
/// </summary>
private const int MaxDestroyedShards = 10; // TODO make configurable
/// <summary>
/// Number of shards allowed to be destroyed before forcing the program to close.
/// </summary>
private const int MaxDestroyedShards = 10; // TODO make configurable
/// <summary>
/// Number of concurrent shard startups to happen on each check.
/// This value is also used in <see cref="DataRetention"/>.
/// </summary>
public const int MaxConcurrentOperations = 5;
/// <summary>
/// Number of concurrent shard startups to happen on each check.
/// This value is also used in <see cref="DataRetention"/>.
/// </summary>
public const int MaxConcurrentOperations = 5;
/// <summary>
/// Amount of time without a completed background service run before a shard instance
/// is considered "dead" and tasked to be removed.
/// </summary>
private static readonly TimeSpan DeadShardThreshold = new(0, 20, 0);
/// <summary>
/// Amount of time without a completed background service run before a shard instance
/// is considered "dead" and tasked to be removed. A fraction of this value is also used
/// to determine when a shard is "slow".
/// </summary>
private static readonly TimeSpan DeadShardThreshold = new(0, 20, 0);
/// <summary>
/// A dictionary with shard IDs as its keys and shard instances as its values.
/// When initialized, all keys will be created as configured. If an instance is removed,
/// a key's corresponding value will temporarily become null instead of the key/value
/// pair being removed.
/// </summary>
private readonly Dictionary<int, ShardInstance> _shards;
/// <summary>
/// A dictionary with shard IDs as its keys and shard instances as its values.
/// When initialized, all keys will be created as configured. If an instance is removed,
/// a key's corresponding value will temporarily become null instead of the key/value
/// pair being removed.
/// </summary>
private readonly Dictionary<int, ShardInstance> _shards;
// Commonly used command handler instances
private readonly Dictionary<string, CommandHandler> _dispatchCommands;
private readonly UserCommands _cmdsUser;
private readonly ListingCommands _cmdsListing;
private readonly HelpInfoCommands _cmdsHelp;
private readonly ManagerCommands _cmdsMods;
private readonly Dictionary<string, CommandHandler> _dispatchCommands;
private readonly UserCommands _cmdsUser;
private readonly ListingCommands _cmdsListing;
private readonly HelpInfoCommands _cmdsHelp;
private readonly ManagerCommands _cmdsMods;
// Watchdog stuff
private readonly Task _watchdogTask;
private readonly CancellationTokenSource _watchdogCancel;
private int _destroyedShards = 0;
internal Configuration Config { get; }
private readonly Task _statusTask;
private readonly CancellationTokenSource _mainCancel;
private int _destroyedShards = 0;
public ShardManager(Configuration cfg)
{
var ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
Log($"Birthday Bot v{ver.ToString(3)} is starting...");
internal Configuration Config { get; }
Config = cfg;
public ShardManager(Configuration cfg) {
var ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
Log($"Birthday Bot v{ver!.ToString(3)} is starting...");
// Command handler setup
_dispatchCommands = new Dictionary<string, CommandHandler>(StringComparer.OrdinalIgnoreCase);
_cmdsUser = new UserCommands(cfg);
foreach (var item in _cmdsUser.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsListing = new ListingCommands(cfg);
foreach (var item in _cmdsListing.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsHelp = new HelpInfoCommands(cfg);
foreach (var item in _cmdsHelp.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsMods = new ManagerCommands(cfg, _cmdsUser.Commands);
foreach (var item in _cmdsMods.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
Config = cfg;
_shards = new Dictionary<int, ShardInstance>();
// Create only the specified shards as needed by this instance
for (int i = Config.ShardStart; i < (Config.ShardStart + Config.ShardAmount); i++)
{
_shards.Add(i, null);
}
// Command handler setup
_dispatchCommands = new Dictionary<string, CommandHandler>(StringComparer.OrdinalIgnoreCase);
_cmdsUser = new UserCommands(cfg);
foreach (var item in _cmdsUser.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsListing = new ListingCommands(cfg);
foreach (var item in _cmdsListing.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsHelp = new HelpInfoCommands(cfg);
foreach (var item in _cmdsHelp.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
_cmdsMods = new ManagerCommands(cfg, _cmdsUser.Commands);
foreach (var item in _cmdsMods.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
// Start watchdog
_watchdogCancel = new CancellationTokenSource();
_watchdogTask = Task.Factory.StartNew(WatchdogLoop, _watchdogCancel.Token,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
// Allocate shards based on configuration
_shards = new Dictionary<int, ShardInstance>();
for (int i = Config.ShardStart; i < (Config.ShardStart + Config.ShardAmount); i++) {
_shards.Add(i, null);
}
public void Dispose()
{
_watchdogCancel.Cancel();
_watchdogTask.Wait(5000);
if (!_watchdogTask.IsCompleted)
Log("Warning: Shard status watcher has not ended in time. Continuing...");
Log("Shutting down all shards...");
var shardDisposes = new List<Task>();
foreach (var item in _shards)
{
if (item.Value == null) continue;
shardDisposes.Add(Task.Run(item.Value.Dispose));
}
if (!Task.WhenAll(shardDisposes).Wait(60000))
{
Log("Warning: All shards did not properly stop after 60 seconds. Continuing...");
}
Log($"Shutdown complete. Bot uptime: {Common.BotUptime}");
}
private void Log(string message) => Program.Log(nameof(ShardManager), message);
/// <summary>
/// Creates and sets up a new shard instance.
/// </summary>
private async Task<ShardInstance> InitializeShard(int shardId)
{
ShardInstance newInstance;
var clientConf = new DiscordSocketConfig()
{
ShardId = shardId,
TotalShards = Config.ShardTotal,
LogLevel = LogSeverity.Info,
DefaultRetryMode = RetryMode.RetryRatelimit,
MessageCacheSize = 0, // not needed at all
GatewayIntents = GatewayIntents.Guilds | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages
};
var newClient = new DiscordSocketClient(clientConf);
newInstance = new ShardInstance(this, newClient, _dispatchCommands);
await newInstance.StartAsync().ConfigureAwait(false);
return newInstance;
}
private async Task WatchdogLoop()
{
try
{
while (!_watchdogCancel.IsCancellationRequested)
{
Log($"Bot uptime: {Common.BotUptime}");
// Iterate through shard list, extract data
var guildInfo = new Dictionary<int, (int, TimeSpan, string)>();
var now = DateTimeOffset.UtcNow;
var nullShards = new List<int>();
foreach (var item in _shards)
{
if (item.Value == null)
{
nullShards.Add(item.Key);
continue;
}
var shard = item.Value;
var guildCount = shard.DiscordClient.Guilds.Count;
var lastRun = now - shard.LastBackgroundRun;
var lastExec = shard.CurrentExecutingService ?? "null";
guildInfo[item.Key] = (guildCount, lastRun, lastExec);
}
// Process info
var guildCounts = guildInfo.Select(i => i.Value.Item1);
var guildTotal = guildCounts.Sum();
var guildAverage = guildCounts.Any() ? guildCounts.Average() : 0;
Log($"Currently in {guildTotal} guilds. Average shard load: {guildAverage:0.0}.");
// Health report
var goodShards = new List<int>();
var badShards = new List<int>(); // shards with low connection score OR long time since last work
var deadShards = new List<int>(); // shards to destroy and reinitialize
foreach (var item in guildInfo)
{
var lastRun = item.Value.Item2;
if (lastRun > DeadShardThreshold / 3)
{
badShards.Add(item.Key);
// Consider a shard dead after a long span without background activity
if (lastRun > DeadShardThreshold)
deadShards.Add(item.Key);
}
else
{
goodShards.Add(item.Key);
}
}
string statusDisplay(IEnumerable<int> list, bool detailedInfo)
{
if (!list.Any()) return "--";
var result = new StringBuilder();
foreach (var item in list)
{
result.Append(item.ToString("00") + " ");
if (detailedInfo)
{
result.Remove(result.Length - 1, 1);
result.Append($"[{Math.Floor(guildInfo[item].Item2.TotalSeconds):000}s");
result.Append($" {guildInfo[item].Item3}] ");
}
}
if (result.Length > 0) result.Remove(result.Length - 1, 1);
return result.ToString();
}
Log("Stable shards: " + statusDisplay(goodShards, false));
if (badShards.Count > 0) Log("Unstable shards: " + statusDisplay(badShards, true));
if (deadShards.Count > 0) Log("Shards to be restarted: " + statusDisplay(deadShards, false));
if (nullShards.Count > 0) Log("Inactive shards: " + statusDisplay(nullShards, false));
// Remove dead shards
foreach (var dead in deadShards) {
// TODO investigate - has this been hanging here?
_shards[dead].Dispose();
_shards[dead] = null;
_destroyedShards++;
}
if (Config.QuitOnFails && _destroyedShards > MaxDestroyedShards)
{
Program.ProgramStop();
}
else
{
// Start up any missing shards
int startAllowance = MaxConcurrentOperations;
foreach (var id in nullShards)
{
// To avoid possible issues with resources strained over so many shards starting at once,
// initialization is spread out by only starting a few at a time.
if (startAllowance-- > 0)
{
_shards[id] = await InitializeShard(id).ConfigureAwait(false);
}
else break;
}
}
// All done for now
await Task.Delay(WatchdogInterval * 1000, _watchdogCancel.Token).ConfigureAwait(false);
}
}
catch (TaskCanceledException) { }
}
// Start status reporting thread
_mainCancel = new CancellationTokenSource();
_statusTask = Task.Factory.StartNew(StatusLoop, _mainCancel.Token,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
}
public void Dispose() {
_mainCancel.Cancel();
_statusTask.Wait(10000);
if (!_statusTask.IsCompleted)
Log("Warning: Main thread did not cleanly finish up in time. Continuing...");
Log("Shutting down all shards...");
var shardDisposes = new List<Task>();
foreach (var item in _shards) {
if (item.Value == null) continue;
shardDisposes.Add(Task.Run(item.Value.Dispose));
}
if (!Task.WhenAll(shardDisposes).Wait(30000)) {
Log("Warning: Not all shards terminated cleanly after 30 seconds. Continuing...");
}
Log($"Uptime: {Common.BotUptime}");
}
private void Log(string message) => Program.Log(nameof(ShardManager), message);
/// <summary>
/// Creates and sets up a new shard instance.
/// </summary>
private async Task<ShardInstance> InitializeShard(int shardId) {
ShardInstance newInstance;
var clientConf = new DiscordSocketConfig() {
ShardId = shardId,
TotalShards = Config.ShardTotal,
LogLevel = LogSeverity.Info,
DefaultRetryMode = RetryMode.RetryRatelimit,
MessageCacheSize = 0, // not needed at all
GatewayIntents = GatewayIntents.Guilds | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages
};
var newClient = new DiscordSocketClient(clientConf);
newInstance = new ShardInstance(this, newClient, _dispatchCommands);
await newInstance.StartAsync().ConfigureAwait(false);
return newInstance;
}
#region Status checking and display
private struct GuildStatusData {
public int GuildCount;
public TimeSpan LastTaskRunTime;
public string? ExecutingTask;
}
private string StatusDisplay(IEnumerable<int> guildList, Dictionary<int, GuildStatusData> guildInfo, bool showDetail) {
if (!guildList.Any()) return "--";
var result = new StringBuilder();
foreach (var item in guildList) {
result.Append(item.ToString("00") + " ");
if (showDetail) {
result.Remove(result.Length - 1, 1);
result.Append($"[{Math.Floor(guildInfo[item].LastTaskRunTime.TotalSeconds):000}s");
if (guildInfo[item].ExecutingTask != null)
result.Append($" {guildInfo[item].ExecutingTask}");
result.Append("] ");
}
}
if (result.Length > 0) result.Remove(result.Length - 1, 1);
return result.ToString();
}
private async Task StatusLoop() {
try {
while (!_mainCancel.IsCancellationRequested) {
Log($"Bot uptime: {Common.BotUptime}");
// Iterate through shard list, extract data
var guildInfo = new Dictionary<int, GuildStatusData>();
var now = DateTimeOffset.UtcNow;
var nullShards = new List<int>();
foreach (var item in _shards) {
if (item.Value == null) {
nullShards.Add(item.Key);
continue;
}
var shard = item.Value;
guildInfo[item.Key] = new GuildStatusData {
GuildCount = shard.DiscordClient.Guilds.Count,
LastTaskRunTime = now - shard.LastBackgroundRun,
ExecutingTask = shard.CurrentExecutingService
};
}
// Process info
var guildCounts = guildInfo.Select(i => i.Value.GuildCount);
var guildTotal = guildCounts.Sum();
var guildAverage = guildCounts.Any() ? guildCounts.Average() : 0;
Log($"Currently in {guildTotal} guilds. Average shard load: {guildAverage:0.0}.");
// Health report
var goodShards = new List<int>();
var badShards = new List<int>(); // shards with low connection score OR long time since last work
var deadShards = new List<int>(); // shards to destroy and reinitialize
foreach (var item in guildInfo) {
var lastRun = item.Value.LastTaskRunTime;
if (lastRun > DeadShardThreshold / 3) {
badShards.Add(item.Key);
// Consider a shard dead after a long span without background activity
if (lastRun > DeadShardThreshold)
deadShards.Add(item.Key);
} else {
goodShards.Add(item.Key);
}
}
Log("Online: " + StatusDisplay(goodShards, guildInfo, false));
if (badShards.Count > 0) Log("Slow: " + StatusDisplay(badShards, guildInfo, true));
if (deadShards.Count > 0) Log("Dead: " + StatusDisplay(deadShards, guildInfo, false));
if (nullShards.Count > 0) Log("Offline: " + StatusDisplay(nullShards, guildInfo, false));
// Remove dead shards
foreach (var dead in deadShards) {
_shards[dead].Dispose();
_shards[dead] = null;
_destroyedShards++;
}
if (Config.QuitOnFails && _destroyedShards > MaxDestroyedShards) {
Environment.ExitCode = 0x04;
Program.ProgramStop();
} else {
// Start up any missing shards
int startAllowance = MaxConcurrentOperations;
foreach (var id in nullShards) {
// To avoid possible issues with resources strained over so many shards starting at once,
// initialization is spread out by only starting a few at a time.
if (startAllowance-- > 0) {
_shards[id] = await InitializeShard(id).ConfigureAwait(false);
} else break;
}
}
await Task.Delay(StatusInterval * 1000, _mainCancel.Token).ConfigureAwait(false);
}
} catch (TaskCanceledException) { }
}
#endregion
}