BirthdayBot/ShardManager.cs

using BirthdayBot.BackgroundServices;
using BirthdayBot.UserInterface;
using Discord;
using Discord.WebSocket;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using static BirthdayBot.UserInterface.CommandsCommon;

namespace BirthdayBot
{
    /// <summary>
    /// The highest level part of this bot:
    /// Starts up, looks over, and manages shard instances while containing common resources
    /// and providing common functions for all existing shards.
    /// </summary>
    class ShardManager : IDisposable
    {
        /// <summary>
        /// Array indexes correspond to shard IDs. Lock on itself when modifying.
        /// </summary>
        private readonly ShardInstance[] _shards;

        // Commonly used command handler instances
        private readonly Dictionary<string, CommandHandler> _dispatchCommands;
        private readonly UserCommands _cmdsUser;
        private readonly ListingCommands _cmdsListing;
        private readonly HelpInfoCommands _cmdsHelp;
        private readonly ManagerCommands _cmdsMods;

        // Watchdog stuff
        private readonly Task _watchdogTask;
        private readonly CancellationTokenSource _watchdogCancel;
        
        internal Configuration Config { get; }

        public ShardManager(Configuration cfg)
        {
            var ver = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
            Log($"Birthday Bot v{ver.ToString(3)} is starting...");

            Config = cfg;

            // Command handler setup
            _dispatchCommands = new Dictionary<string, CommandHandler>(StringComparer.OrdinalIgnoreCase);
            _cmdsUser = new UserCommands(cfg);
            foreach (var item in _cmdsUser.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
            _cmdsListing = new ListingCommands(cfg);
            foreach (var item in _cmdsListing.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
            _cmdsHelp = new HelpInfoCommands(cfg);
            foreach (var item in _cmdsHelp.Commands) _dispatchCommands.Add(item.Item1, item.Item2);
            _cmdsMods = new ManagerCommands(cfg, _cmdsUser.Commands);
            foreach (var item in _cmdsMods.Commands) _dispatchCommands.Add(item.Item1, item.Item2);

            // Start shards
            _shards = new ShardInstance[Config.ShardCount];
            for (int i = 0; i < _shards.Length; i++)
                InitializeShard(i).Wait();

            // Start watchdog
            _watchdogCancel = new CancellationTokenSource();
            _watchdogTask = Task.Factory.StartNew(WatchdogLoop, _watchdogCancel.Token,
                                                  TaskCreationOptions.LongRunning, TaskScheduler.Default);
        }

        public void Dispose()
        {
            Log("Captured cancel key. Shutting down shard status watcher...");
            _watchdogCancel.Cancel();
            _watchdogTask.Wait(5000);
            if (!_watchdogTask.IsCompleted)
                Log("Warning: Shard status watcher has not ended in time. Continuing...");

            Log("Shutting down all shards...");
            var shardDisposes = new List<Task>();
            foreach (var shard in _shards)
            {
                if (shard == null) continue;
                shardDisposes.Add(Task.Run(shard.Dispose));
            }
            if (!Task.WhenAll(shardDisposes).Wait(60000))
            {
                Log("Warning: All shards did not properly stop after 60 seconds. Continuing...");
            }

            Log($"Shutdown complete. Bot uptime: {Common.BotUptime}");
        }

        private void Log(string message) => Program.Log(nameof(ShardManager), message);

        /// <summary>
        /// Creates and sets up a new shard instance.
        /// </summary>
        private async Task InitializeShard(int shardId)
        {
            ShardInstance newInstance;
            lock (_shards)
            {
                var clientConf = new DiscordSocketConfig()
                {
                    ShardId = shardId,
                    TotalShards = Config.ShardCount,
                    LogLevel = LogSeverity.Info,
                    DefaultRetryMode = RetryMode.RetryRatelimit,
                    MessageCacheSize = 0, // not needed at all
                    ExclusiveBulkDelete = true, // not relevant, but this is configured to skip the warning
                    AlwaysDownloadUsers = true, // TODO set to false when more stable to do so
                    GatewayIntents = GatewayIntents.Guilds | GatewayIntents.GuildMembers | GatewayIntents.GuildMessages
                };
                var newClient = new DiscordSocketClient(clientConf);
                newInstance = new ShardInstance(this, newClient, _dispatchCommands);

                _shards[shardId] = newInstance;
            }
            await newInstance.StartAsync().ConfigureAwait(false);
        }

        private async Task WatchdogLoop()
        {
            try
            {
                while (!_watchdogCancel.IsCancellationRequested)
                {
                    Log($"Bot uptime: {Common.BotUptime}");

                    // Gather statistical information within the lock
                    var guildInfo = new (int, int, TimeSpan)[_shards.Length]; // guild count, conn score, last run
                    var now = DateTimeOffset.UtcNow;
                    ulong? botId = null;
                    lock (_shards)
                    {
                        for (int i = 0; i < _shards.Length; i++)
                        {
                            var shard = _shards[i];
                            botId ??= shard.DiscordClient.CurrentUser?.Id;

                            var guildCount = shard.DiscordClient.Guilds.Count;
                            var connScore = shard.ConnectionScore;
                            var lastRun = now - shard.LastBackgroundRun;
                            guildInfo[i] = (guildCount, connScore, lastRun);
                        }
                    }

                    // Process info
                    var guildCounts = guildInfo.Select(i => i.Item1);
                    var guildTotal = guildCounts.Sum();
                    var guildAverage = guildCounts.Average();
                    Log($"Currently in {guildTotal} guilds. Average shard load: {guildAverage:0.0}.");
                    if (botId.HasValue)
                        await SendExternalStatistics(guildTotal, botId.Value, _watchdogCancel.Token).ConfigureAwait(false);

                    // Health report
                    var goodShards = new List<int>();
                    var badShards = new List<int>(); // shards with a low connection score / long time since last work
                    var deadShards = new List<int>(); // shards to destroy and reinitialize
                    for (int i = 0; i < guildInfo.Length; i++)
                    {
                        var connScore = guildInfo[i].Item2;
                        var lastRun = guildInfo[i].Item3;

                        if (lastRun > new TimeSpan(0, 20, 0) || connScore < ConnectionStatus.StableScore)
                        {
                            badShards.Add(i);

                            // This is for now the only deciding factor on whether to discard a shard,
                            // without regards to score.
                            if (lastRun > new TimeSpan(1, 0, 0))
                            {
                                deadShards.Add(i);
                            }
                        }
                        else
                        {
                            goodShards.Add(i);
                        }
                    }

                    string catNumbers(IEnumerable<int> list, bool detailedInfo)
                    {
                        if (!list.Any()) return "--";
                        var result = new StringBuilder();
                        foreach (var item in list)
                        {
                            result.Append(item.ToString("00") + " ");
                            if (detailedInfo)
                            {
                                result.Remove(result.Length - 1, 1);
                                result.Append($"[{guildInfo[item].Item2:+000;-000}");
                                result.Append($" {Math.Floor(guildInfo[item].Item3.TotalMinutes):00}m");
                                result.Append($"{guildInfo[item].Item3.Seconds:00}s] ");
                            }
                            
                        }
                        if (result.Length > 0) result.Remove(result.Length - 1, 1);
                        return result.ToString();
                    }
                    Log("Stable shards: " + catNumbers(goodShards, false));
                    if (badShards.Count > 0) Log("Unstable shards: " + catNumbers(badShards, true));
                    if (deadShards.Count > 0) Log("Shards to be restarted: " + catNumbers(deadShards, false));
                    {

                    }

                    // 120 second delay
                    await Task.Delay(120 * 1000, _watchdogCancel.Token).ConfigureAwait(false);
                }
            }
            catch (TaskCanceledException) { }
        }

        #region Statistical reporting
        private static readonly HttpClient _httpClient = new HttpClient();

        /// <summary>
        /// Send statistical information to external services.
        /// </summary>
        private async Task SendExternalStatistics(int count, ulong botId, CancellationToken token)
        {
            var dbotsToken = Config.DBotsToken;
            if (dbotsToken != null)
            {
                try
                {
                    const string dBotsApiUrl = "https://discord.bots.gg/api/v1/bots/{0}/stats";
                    const string Body = "{{ \"guildCount\": {0} }}";
                    var uri = new Uri(string.Format(dBotsApiUrl, botId));

                    var post = new HttpRequestMessage(HttpMethod.Post, uri);
                    post.Headers.Add("Authorization", dbotsToken);
                    post.Content = new StringContent(string.Format(Body, count), Encoding.UTF8, "application/json");

                    await _httpClient.SendAsync(post, token);
                    Log("Discord Bots: Update successful.");
                }
                catch (Exception ex)
                {
                    Log("Discord Bots: Exception encountered during update: " + ex.Message);
                }
            }
        }
        #endregion
    }
}