From b67716ec946fc9f41187643ead8805b65a8ffe6f Mon Sep 17 00:00:00 2001 From: Noikoio Date: Sun, 5 Nov 2017 20:55:57 -0800 Subject: [PATCH] Finished implementing database caches --- BotFeature.cs | 2 +- Feature/DBCache/EntityCache.cs | 140 +++++++++++++++++++------------- Feature/DBCache/MessageCache.cs | 92 +++++++++++++-------- RegexBot.cs | 4 +- 4 files changed, 142 insertions(+), 96 deletions(-) diff --git a/BotFeature.cs b/BotFeature.cs index deb9c57..ca43b8f 100644 --- a/BotFeature.cs +++ b/BotFeature.cs @@ -20,7 +20,7 @@ namespace Noikoio.RegexBot public abstract string Name { get; } protected DiscordSocketClient Client => _client; - protected BotFeature(DiscordSocketClient client) + public BotFeature(DiscordSocketClient client) { _client = client; _logger = Logger.GetLogger(this.Name); diff --git a/Feature/DBCache/EntityCache.cs b/Feature/DBCache/EntityCache.cs index a92bd62..26b221f 100644 --- a/Feature/DBCache/EntityCache.cs +++ b/Feature/DBCache/EntityCache.cs @@ -1,6 +1,7 @@ using Discord.WebSocket; using Newtonsoft.Json.Linq; using Noikoio.RegexBot.ConfigItem; +using Npgsql; using NpgsqlTypes; using System; using System.Collections.Generic; @@ -24,6 +25,8 @@ namespace Noikoio.RegexBot.Feature.DBCache if (_db.Enabled) { + CreateCacheTables(); + client.GuildAvailable += Client_GuildAvailable; client.GuildUpdated += Client_GuildUpdated; client.GuildMemberUpdated += Client_GuildMemberUpdated; @@ -41,10 +44,12 @@ namespace Noikoio.RegexBot.Feature.DBCache // Guild _and_ guild member information has become available private async Task Client_GuildAvailable(SocketGuild arg) { - await CreateCacheTables(arg.Id); - - await Task.Run(() => UpdateGuild(arg)); - await Task.Run(() => UpdateGuildMember(arg.Id, arg.Users)); + await Task.Run(async () => + { + await UpdateGuild(arg); + await UpdateGuildMember(arg.Users); + } + ); } // Guild information has changed @@ -58,15 +63,15 @@ namespace Noikoio.RegexBot.Feature.DBCache { await Task.Run(() => UpdateGuildMember(arg2)); } - #endregion +#endregion - #region Table setup +#region Table setup public const string TableGuild = "cache_guild"; - const string TableUser = "cache_users"; + public const string TableUser = "cache_users"; - private async Task CreateCacheTables(ulong gid) + private void CreateCacheTables() { - using (var db = await _db.GetOpenConnectionAsync()) + using (var db = _db.GetOpenConnectionAsync().GetAwaiter().GetResult()) { using (var c = db.CreateCommand()) { @@ -75,89 +80,110 @@ namespace Noikoio.RegexBot.Feature.DBCache + "current_name text not null, " + "display_name text null" + ")"; - // TODO determine if other columns necessary? - await c.ExecuteNonQueryAsync(); + // TODO determine if other columns might be needed? + c.ExecuteNonQuery(); } using (var c = db.CreateCommand()) { c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableUser + "(" - + "user_id bigint, " - + "guild_id bigint references " + TableGuild + " (guild_id), " + + "user_id bigint not null, " + + $"guild_id bigint not null references {TableGuild}, " + "cache_date timestamptz not null, " + "username text not null, " + "discriminator text not null, " + "nickname text null, " + "avatar_url text null" + ")"; - await c.ExecuteNonQueryAsync(); + c.ExecuteNonQuery(); } using (var c = db.CreateCommand()) { c.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS " + $"{TableUser}_idx on {TableUser} (user_id, guild_id)"; - await c.ExecuteNonQueryAsync(); + c.ExecuteNonQuery(); } } } - #endregion +#endregion private async Task UpdateGuild(SocketGuild g) { - using (var db = await _db.GetOpenConnectionAsync()) + try { - using (var c = db.CreateCommand()) + using (var db = await _db.GetOpenConnectionAsync()) { - c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) " - + "(@GuildId, @CurrentName) " - + "ON CONFLICT (guild_id) DO UPDATE SET " - + "current_name = EXCLUDED.current_name"; - c.Parameters.Add("@GuildID", NpgsqlDbType.Bigint).Value = g.Id; - c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name; - c.Prepare(); - await c.ExecuteNonQueryAsync(); - } - } - } - - private async Task UpdateGuildMember(ulong gid, IEnumerable users) - { - using (var db = await _db.GetOpenConnectionAsync()) - { - using (var c = db.CreateCommand()) - { - c.CommandText = "INSERT INTO " + TableUser + " VALUES " - + "(@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) " - + "ON CONFLICT (user_id, guild_id) DO UPDATE SET " - + "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, " - + "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time... - + "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url"; - c.Prepare(); - - var now = DateTime.Now; - List inserts = new List(); - - foreach (var item in users) + using (var c = db.CreateCommand()) { - c.Parameters.Clear(); - c.Parameters.Add("@Uid", NpgsqlDbType.Bigint).Value = item.Id; - c.Parameters.Add("@Gid", NpgsqlDbType.Bigint).Value = item.Guild.Id; - c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = now; - c.Parameters.Add("@Uname", NpgsqlDbType.Text).Value = item.Username; - c.Parameters.Add("@Disc", NpgsqlDbType.Text).Value = item.Discriminator; - c.Parameters.Add("@Nname", NpgsqlDbType.Text).Value = item.Nickname; - c.Parameters.Add("@Url", NpgsqlDbType.Text).Value = item.GetAvatarUrl(); + c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) " + + "VALUES (@GuildId, @CurrentName) " + + "ON CONFLICT (guild_id) DO UPDATE SET " + + "current_name = EXCLUDED.current_name"; + c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = g.Id; + c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name; + c.Prepare(); await c.ExecuteNonQueryAsync(); } } } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(UpdateGuild)}: " + ex.Message); + } + } + + private async Task UpdateGuildMember(IEnumerable users) + { + try + { + using (var db = await _db.GetOpenConnectionAsync()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "INSERT INTO " + TableUser + + " (user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url)" + + " VALUES (@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) " + + "ON CONFLICT (user_id, guild_id) DO UPDATE SET " + + "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, " + + "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time... + + "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url"; + + var uid = c.Parameters.Add("@Uid", NpgsqlDbType.Bigint); + var gid = c.Parameters.Add("@Gid", NpgsqlDbType.Bigint); + c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now; + var uname = c.Parameters.Add("@Uname", NpgsqlDbType.Text); + var disc = c.Parameters.Add("@Disc", NpgsqlDbType.Text); + var nname = c.Parameters.Add("@Nname", NpgsqlDbType.Text); + var url = c.Parameters.Add("@Url", NpgsqlDbType.Text); + c.Prepare(); + + foreach (var item in users) + { + uid.Value = item.Id; + gid.Value = item.Guild.Id; + uname.Value = item.Username; + disc.Value = item.Discriminator; + nname.Value = item.Nickname; + if (nname.Value == null) nname.Value = DBNull.Value; // why can't ?? work here? + url.Value = item.GetAvatarUrl(); + if (url.Value == null) url.Value = DBNull.Value; + + await c.ExecuteNonQueryAsync(); + } + } + } + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(UpdateGuildMember)}: " + ex.Message); + } } private Task UpdateGuildMember(SocketGuildUser user) { var gid = user.Guild.Id; var ml = new SocketGuildUser[] { user }; - return UpdateGuildMember(gid, ml); + return UpdateGuildMember(ml); } } } diff --git a/Feature/DBCache/MessageCache.cs b/Feature/DBCache/MessageCache.cs index f2c4b9a..a8b6fb4 100644 --- a/Feature/DBCache/MessageCache.cs +++ b/Feature/DBCache/MessageCache.cs @@ -1,12 +1,11 @@ using Discord.WebSocket; using Newtonsoft.Json.Linq; using Noikoio.RegexBot.ConfigItem; -using System; -using System.Collections.Generic; -using System.Text; +using Npgsql; +using NpgsqlTypes; using System.Threading.Tasks; -namespace Noikoio.RegexBot.Feature.DBCaches +namespace Noikoio.RegexBot.Feature.DBCache { /// /// Caches information regarding all incoming messages. @@ -14,31 +13,38 @@ namespace Noikoio.RegexBot.Feature.DBCaches /// class MessageCache : BotFeature { + // TODO Something that clears expired cache items private readonly DatabaseConfig _db; public override string Name => nameof(MessageCache); - #region Table setup - const string TableGuild = "cache_guild"; - const string TableUser = "cache_users"; - const string TableMessage = "cache_messages"; - public MessageCache(DiscordSocketClient client) : base(client) { _db = RegexBot.Config.Database; - client.MessageReceived += Client_MessageReceived; - //client.MessageUpdated += Client_MessageUpdated; + if (_db.Enabled) + { + CreateCacheTables(); + + client.MessageReceived += Client_MessageReceived; + //client.MessageUpdated += Client_MessageUpdated; + } + else + { + Log("No database storage available.").Wait(); + } } + #region Table setup + const string TableMessage = "cache_messages"; + public override Task ProcessConfiguration(JToken configSection) => Task.FromResult(null); #region Event handling // A new message has been created private async Task Client_MessageReceived(SocketMessage arg) { - if (!_db.Enabled) return; - throw new NotImplementedException(); + await Task.Run(() => CacheMessage(arg)); } //private Task Client_MessageUpdated(Discord.Cacheable arg1, SocketMessage arg2, ISocketMessageChannel arg3) @@ -50,28 +56,24 @@ namespace Noikoio.RegexBot.Feature.DBCaches */ #endregion - private async Task CreateCacheTables(ulong gid) + private void CreateCacheTables() { - /* Note: - * We save information per guild in their own schemas named "g_NUM", where NUM is the Guild ID. - * - * The creation of these schemas is handled within here, but we're possibly facing a short delay - * in the event that other events that we're listening for come in without a schema having been - * created yet in which to put them in. - * Got to figure that out. - */ - await _db.CreateGuildSchemaAsync(gid); - - using (var db = await _db.OpenConnectionAsync(gid)) + using (var db = _db.GetOpenConnectionAsync().GetAwaiter().GetResult()) { using (var c = db.CreateCommand()) { - c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableMessage + "(" - + "snowflake bigint primary key, " - + "cache_date timestamptz not null, " - + "author bigint not null" + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableMessage + " (" + + "message_id bigint primary key, " + + "author_id bigint not null, " + + "guild_id bigint not null, " + + "channel_id bigint not null, " // channel cache later? something to think about... + + "created_ts timestamptz not null, " + + "edited_ts timestamptz null, " + + "message text not null, " + + $"FOREIGN KEY (author_id, guild_id) references {EntityCache.TableUser} (user_id, guild_id)" + ")"; - await c.ExecuteNonQueryAsync(); + // TODO figure out how to store message edits + c.ExecuteNonQuery(); } } } @@ -79,12 +81,30 @@ namespace Noikoio.RegexBot.Feature.DBCaches private async Task CacheMessage(SocketMessage msg) { - throw new NotImplementedException(); - } - - private async Task UpdateMessage(SocketMessage msg) - { - throw new NotImplementedException(); + try + { + using (var db = await _db.GetOpenConnectionAsync()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "INSERT INTO " + TableMessage + + " (message_id, author_id, guild_id, channel_id, created_ts, message) VALUES " + + "(@MessageId, @UserId, @GuildId, @ChannelId, @Date, @Message)"; + c.Parameters.Add("@MessageId", NpgsqlDbType.Bigint).Value = msg.Id; + c.Parameters.Add("@UserId", NpgsqlDbType.Bigint).Value = msg.Author.Id; + c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = ((SocketGuildUser)msg.Author).Guild.Id; + c.Parameters.Add("@ChannelId", NpgsqlDbType.Bigint).Value = msg.Channel.Id; + c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = msg.Timestamp; + c.Parameters.Add("@Message", NpgsqlDbType.Text).Value = msg.Content; + c.Prepare(); + await c.ExecuteNonQueryAsync(); + } + } + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(CacheMessage)}: " + ex.Message); + } } } } diff --git a/RegexBot.cs b/RegexBot.cs index 25f5fd7..5f36039 100644 --- a/RegexBot.cs +++ b/RegexBot.cs @@ -36,7 +36,7 @@ namespace Noikoio.RegexBot LogLevel = LogSeverity.Info, AlwaysDownloadUsers = true, DefaultRetryMode = RetryMode.AlwaysRetry, - MessageCacheSize = 50 + MessageCacheSize = 0 }); // Hook up handlers for basic functions @@ -48,7 +48,7 @@ namespace Noikoio.RegexBot new Feature.AutoMod.AutoMod(_client), new Feature.ModTools.ModTools(_client), new Feature.AutoRespond.AutoRespond(_client), - new Feature.DBCache.EntityCache(_client), + new Feature.DBCache.EntityCache(_client), // EntityCache goes before anything else that uses its data new Feature.DBCache.MessageCache(_client) }; var dlog = Logger.GetLogger("Discord.Net");