From 06e559b085ac892701417ca646b45e5d9147ea37 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Tue, 14 Nov 2017 15:32:18 -0800 Subject: [PATCH 01/13] First commit of ModLogs Some files added. At the moment everyhing is mostly rough plans. --- Module/ModLogs/Entry.cs | 57 +++++++++++++++++ Module/ModLogs/EventListener.cs | 13 ++++ Module/ModLogs/MessageCache.cs | 110 ++++++++++++++++++++++++++++++++ Module/ModLogs/ModLogs.cs | 10 +++ 4 files changed, 190 insertions(+) create mode 100644 Module/ModLogs/Entry.cs create mode 100644 Module/ModLogs/EventListener.cs create mode 100644 Module/ModLogs/MessageCache.cs create mode 100644 Module/ModLogs/ModLogs.cs diff --git a/Module/ModLogs/Entry.cs b/Module/ModLogs/Entry.cs new file mode 100644 index 0000000..c3bec89 --- /dev/null +++ b/Module/ModLogs/Entry.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Noikoio.RegexBot.Module.ModLogs +{ + /// + /// Represents a log entry. + /// + class Entry + { + readonly int _logId; + readonly DateTime _ts; + readonly ulong? _invokeId; + readonly ulong _targetId; + readonly ulong? _channelId; + readonly string _type; + readonly string _message; + + /// + /// Gets the ID value of this log entry. + /// + public int Id => _logId; + /// + /// Gets the timestamp (a with ) of the entry. + /// + public DateTime Timestamp => _ts; + /// + /// Gets the ID of the invoking user. + /// This value exists only if this entry was created through action of another user that is not the target. + /// + public ulong? Invoker => _invokeId; + /// + /// Gets the ID of the user to which this log entry corresponds. + /// + public ulong Target => _targetId; + /// + /// Gets the guild channel ID to which this log entry corresponds, if any. + /// + public ulong? TargetChannel => _channelId; + /// + /// Gets this log entry's 'type', or category. + /// + public string LogType => _type; + /// + /// Gets the content of this log entry. + /// + public string Message => _message; + + public Entry() + { + throw new NotImplementedException(); + } + + // TODO figure out some helper methods to retrieve data of other entities by ID, if it becomes necessary + } +} diff --git a/Module/ModLogs/EventListener.cs b/Module/ModLogs/EventListener.cs new file mode 100644 index 0000000..3984696 --- /dev/null +++ b/Module/ModLogs/EventListener.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Noikoio.RegexBot.Module.ModLogs +{ + /// + /// Listens for certain events and places them on the log. + /// + class EventListener + { + } +} diff --git a/Module/ModLogs/MessageCache.cs b/Module/ModLogs/MessageCache.cs new file mode 100644 index 0000000..80cd933 --- /dev/null +++ b/Module/ModLogs/MessageCache.cs @@ -0,0 +1,110 @@ +using Discord.WebSocket; +using Newtonsoft.Json.Linq; +using Noikoio.RegexBot.ConfigItem; +using Npgsql; +using NpgsqlTypes; +using System.Threading.Tasks; + +namespace Noikoio.RegexBot.Module.DBCache +{ + /// + /// Caches information regarding all incoming messages. + /// The function of this feature should be transparent to the user, and thus no configuration is needed. + /// + class MessageCache : BotFeature + { + // TODO Something that clears expired cache items + private readonly DatabaseConfig _db; + + public override string Name => nameof(MessageCache); + + public MessageCache(DiscordSocketClient client) : base(client) + { + _db = RegexBot.Config.Database; + + if (_db.Enabled) + { + CreateCacheTables(); + + client.MessageReceived += Client_MessageReceived; + //client.MessageUpdated += Client_MessageUpdated; + } + else + { + Log("No database storage available.").Wait(); + } + } + + #region Table setup + const string TableMessage = "cache_messages"; + + public override Task ProcessConfiguration(JToken configSection) => Task.FromResult(null); + + #region Event handling + // A new message has been created + private async Task Client_MessageReceived(SocketMessage arg) + { + await Task.Run(() => CacheMessage(arg)); + } + + //private Task Client_MessageUpdated(Discord.Cacheable arg1, SocketMessage arg2, ISocketMessageChannel arg3) + /* + * Edited messages seem to retain their ID. This is a problem. + * The point of this message cache was to have another feature be able to relay + * both the previous and current message at once. + * For now: Do nothing on updated messages. + */ + #endregion + + private void CreateCacheTables() + { + using (var db = _db.GetOpenConnectionAsync().GetAwaiter().GetResult()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableMessage + " (" + + "message_id bigint primary key, " + + "author_id bigint not null, " + + "guild_id bigint not null, " + + "channel_id bigint not null, " // channel cache later? something to think about... + + "created_ts timestamptz not null, " + + "edited_ts timestamptz null, " + + "message text not null, " + + $"FOREIGN KEY (author_id, guild_id) references {EntityCache.Sql.TableUser} (user_id, guild_id)" + + ")"; + // TODO figure out how to store message edits + c.ExecuteNonQuery(); + } + } + } + #endregion + + private async Task CacheMessage(SocketMessage msg) + { + try + { + using (var db = await _db.GetOpenConnectionAsync()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "INSERT INTO " + TableMessage + + " (message_id, author_id, guild_id, channel_id, created_ts, message) VALUES " + + "(@MessageId, @UserId, @GuildId, @ChannelId, @Date, @Message)"; + c.Parameters.Add("@MessageId", NpgsqlDbType.Bigint).Value = msg.Id; + c.Parameters.Add("@UserId", NpgsqlDbType.Bigint).Value = msg.Author.Id; + c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = ((SocketGuildUser)msg.Author).Guild.Id; + c.Parameters.Add("@ChannelId", NpgsqlDbType.Bigint).Value = msg.Channel.Id; + c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = msg.Timestamp; + c.Parameters.Add("@Message", NpgsqlDbType.Text).Value = msg.Content; + c.Prepare(); + await c.ExecuteNonQueryAsync(); + } + } + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(CacheMessage)}: " + ex.Message); + } + } + } +} diff --git a/Module/ModLogs/ModLogs.cs b/Module/ModLogs/ModLogs.cs new file mode 100644 index 0000000..14fc93b --- /dev/null +++ b/Module/ModLogs/ModLogs.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Noikoio.RegexBot.Module.ModLogs +{ + class ModLogs + { + } +} \ No newline at end of file From 5d5cdd9c847aa3125368800bb6fd71af3351d2c1 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 17 Nov 2017 10:49:37 -0800 Subject: [PATCH 02/13] Added missing fields to ModLogs entry --- Module/ModLogs/Entry.cs | 11 ++++++++--- Module/ModLogs/EventListener.cs | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Module/ModLogs/Entry.cs b/Module/ModLogs/Entry.cs index c3bec89..3867b32 100644 --- a/Module/ModLogs/Entry.cs +++ b/Module/ModLogs/Entry.cs @@ -11,6 +11,7 @@ namespace Noikoio.RegexBot.Module.ModLogs { readonly int _logId; readonly DateTime _ts; + readonly ulong _guildId; readonly ulong? _invokeId; readonly ulong _targetId; readonly ulong? _channelId; @@ -26,15 +27,19 @@ namespace Noikoio.RegexBot.Module.ModLogs /// public DateTime Timestamp => _ts; /// - /// Gets the ID of the invoking user. - /// This value exists only if this entry was created through action of another user that is not the target. + /// Gets the ID of the guild to which this log entry corresponds. /// - public ulong? Invoker => _invokeId; + public ulong Guild => _guildId; /// /// Gets the ID of the user to which this log entry corresponds. /// public ulong Target => _targetId; /// + /// Gets the ID of the invoking user. + /// This value exists only if this entry was created through action of another user that is not the target. + /// + public ulong? Invoker => _invokeId; + /// /// Gets the guild channel ID to which this log entry corresponds, if any. /// public ulong? TargetChannel => _channelId; diff --git a/Module/ModLogs/EventListener.cs b/Module/ModLogs/EventListener.cs index 3984696..2ed1932 100644 --- a/Module/ModLogs/EventListener.cs +++ b/Module/ModLogs/EventListener.cs @@ -1,13 +1,28 @@ using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; +using Discord.WebSocket; +using Newtonsoft.Json.Linq; namespace Noikoio.RegexBot.Module.ModLogs { /// - /// Listens for certain events and places them on the log. + /// Listens for Discord-based events and writes them to the log (database). + /// Additionally writes certain messages to a designated logging channel if configured. /// - class EventListener + class EventListener : BotModule { + public override string Name => "ModLogs"; + public EventListener(DiscordSocketClient client) : base(client) + { + + } + + [ConfigSection("modlogs")] + public override Task ProcessConfiguration(JToken configSection) + { + throw new NotImplementedException(); + } } } From 0ff7c4c37500101045cc8d290c12a96c68179798 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Mon, 20 Nov 2017 13:21:05 -0800 Subject: [PATCH 03/13] Add extra note --- Module/EntityCache/UserCacheItem.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Module/EntityCache/UserCacheItem.cs b/Module/EntityCache/UserCacheItem.cs index d82ace5..1e94e95 100644 --- a/Module/EntityCache/UserCacheItem.cs +++ b/Module/EntityCache/UserCacheItem.cs @@ -77,6 +77,7 @@ namespace Noikoio.RegexBot.Module.EntityCache public override string ToString() => DisplayName; #region Queries + // Double-check constructor if making changes to this constant const string QueryColumns = "user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url"; /// From 9cfdeba8be216e6772532694ba3d108add9a0439 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Mon, 20 Nov 2017 19:32:34 -0800 Subject: [PATCH 04/13] Added Entry queries --- Module/ModLogs/Entry.cs | 118 ++++++++++++++++++++++++++++++++++++++-- Module/ModLogs/Sql.cs | 47 ++++++++++++++++ 2 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 Module/ModLogs/Sql.cs diff --git a/Module/ModLogs/Entry.cs b/Module/ModLogs/Entry.cs index 3867b32..4e7918b 100644 --- a/Module/ModLogs/Entry.cs +++ b/Module/ModLogs/Entry.cs @@ -1,11 +1,14 @@ -using System; +using Npgsql; +using System; using System.Collections.Generic; +using System.Data.Common; using System.Text; +using System.Threading.Tasks; namespace Noikoio.RegexBot.Module.ModLogs { /// - /// Represents a log entry. + /// Represents a log entry in the database. /// class Entry { @@ -44,19 +47,122 @@ namespace Noikoio.RegexBot.Module.ModLogs /// public ulong? TargetChannel => _channelId; /// - /// Gets this log entry's 'type', or category. + /// Gets this log entry's category. /// - public string LogType => _type; + public string Category => _type; /// /// Gets the content of this log entry. /// public string Message => _message; - public Entry() + public Entry(DbDataReader r) { - throw new NotImplementedException(); + // Double-check ordinals if making changes to QueryColumns + + _logId = r.GetInt32(0); + _ts = r.GetDateTime(1).ToUniversalTime(); + unchecked + { + _guildId = (ulong)r.GetInt64(2); + _targetId = (ulong)r.GetInt64(3); + if (r.IsDBNull(4)) _invokeId = null; + else _invokeId = (ulong)r.GetInt64(4); + if (r.IsDBNull(5)) _channelId = null; + else _channelId = (ulong)r.GetInt64(5); + } + _type = r.GetString(6); + _message = r.GetString(7); } + // TODO lazy loading of channel, user, etc from caches + // TODO methods for updating this log entry(?) + // TODO figure out some helper methods to retrieve data of other entities by ID, if it becomes necessary + + #region Queries + // Double-check constructor if making changes to this constant + const string QueryColumns = "id, entry_ts, guild_id, target_id, invoke_id, target_channel_id, category, message"; + + /// + /// Attempts to look up a log entry with the given ID. + /// + /// Null if no result. + public static async Task QueryIdAsync(ulong guild, int id) + { + using (var db = await RegexBot.Config.Database.GetOpenConnectionAsync()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = $"SELECT {QueryColumns} FROM {Sql.TableLog} " + + "WHERE guild_id = @Guild and id = @Id"; + c.Parameters.Add("@Guild", NpgsqlTypes.NpgsqlDbType.Bigint).Value = guild; + c.Parameters.Add("@Id", NpgsqlTypes.NpgsqlDbType.Numeric).Value = id; + c.Prepare(); + using (var r = await c.ExecuteReaderAsync()) + { + if (r.Read()) return new Entry(r); + else return null; + } + } + } + } + + public static async Task> QueryLogAsync + (ulong guild, + ulong? target = null, + ulong? invoker = null, + ulong? channel = null, + IEnumerable category = null) + { + // Enforce some limits - can't search too broadly here. Requires this at a minimum: + if (target.HasValue == false && invoker.HasValue == false) + { + throw new ArgumentNullException("Query requires at minimum searching of a target or invoker."); + } + + var result = new List(); + using (var db = await RegexBot.Config.Database.GetOpenConnectionAsync()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = $"SELECT {QueryColumns} FROM {Sql.TableLog} WHERE"; + + bool and = false; + if (target.HasValue) + { + if (and) c.CommandText += " AND"; + else and = true; + c.CommandText += " target_id = @TargetId"; + c.Parameters.Add("@TargetId", NpgsqlTypes.NpgsqlDbType.Bigint).Value = target.Value; + } + if (invoker.HasValue) + { + if (and) c.CommandText += " AND"; + else and = true; + c.CommandText += " invoke_id = @InvokeId"; + c.Parameters.Add("@InvokeId", NpgsqlTypes.NpgsqlDbType.Bigint).Value = invoker.Value; + } + if (channel.HasValue) + { + if (and) c.CommandText += " AND"; + else and = true; + c.CommandText += " target_channel_id = @ChannelId"; + c.Parameters.Add("@ChannelId", NpgsqlTypes.NpgsqlDbType.Bigint).Value = channel.Value; + } + c.Prepare(); + + using (var r = await c.ExecuteReaderAsync()) + { + while (r.Read()) + { + result.Add(new Entry(r)); + } + } + } + } + + return result; + } + #endregion } } diff --git a/Module/ModLogs/Sql.cs b/Module/ModLogs/Sql.cs new file mode 100644 index 0000000..c500698 --- /dev/null +++ b/Module/ModLogs/Sql.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Noikoio.RegexBot.Module.ModLogs +{ + /// + /// Contains common constants and static methods used for accessing the log database. + /// + class Sql + { + public const string TableLog = "modlogs_entries"; + public const string TableLogIncr = TableLog + "_id"; + public const string TableMsgCache = "modlogs_msgcache"; + + static void CreateTables() + { + using (var db = RegexBot.Config.Database.GetOpenConnectionAsync().GetAwaiter().GetResult()) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableLog + " (" + + "id int primary key, " + + "entry_ts timestamptz not null, " + + "guild_id bigint not null, " + + "target_id bigint not null, " + + $"invoke_id bigint null references {EntityCache.Sql.TableUser}.user_id, " + + "target_channel_id bigint null, " // TODO channel cache reference? + + "category text not null, " + + "message text not null, " + + $"FOREIGN KEY (target_id, guild_id) REFERENCES {EntityCache.Sql.TableUser} (user_id, guild_id)"; + c.ExecuteNonQuery(); + } + using (var c = db.CreateCommand()) + { + c.CommandText = $"CREATE SEQUENCE IF NOT EXISTS {TableLogIncr} " + + $"START 100 MAXVALUE {int.MaxValue}"; + c.ExecuteNonQuery(); + } + } + } + + #region Log entry manipulation + // what was I doing again + #endregion + } +} From 33e893e2788219d2b0165f58706e8069f78414e2 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 8 Dec 2017 13:52:32 -0800 Subject: [PATCH 05/13] Fix incoming users not getting cached --- Module/EntityCache/EntityCache.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Module/EntityCache/EntityCache.cs b/Module/EntityCache/EntityCache.cs index 8af68ec..36a66df 100644 --- a/Module/EntityCache/EntityCache.cs +++ b/Module/EntityCache/EntityCache.cs @@ -31,18 +31,19 @@ namespace Noikoio.RegexBot.Module.EntityCache client.GuildAvailable += Client_GuildAvailable; client.GuildUpdated += Client_GuildUpdated; client.GuildMemberUpdated += Client_GuildMemberUpdated; - // it may not be necessary to handle JoinedGuild, as GuildAvailable provides this info + client.UserJoined += Client_UserJoined; } else { Log("No database storage available.").Wait(); } } - + public override Task ProcessConfiguration(JToken configSection) => Task.FromResult(null); #region Event handling - // Guild _and_ guild member information has become available + // Guild and guild member information has become available. + // This is a very expensive operation, when joining larger guilds for the first time. private async Task Client_GuildAvailable(SocketGuild arg) { await Task.Run(async () => @@ -64,12 +65,14 @@ namespace Noikoio.RegexBot.Module.EntityCache { await Task.Run(() => UpdateGuildMember(arg2)); } -#endregion -#region Table setup - -#endregion - + // A new guild member has appeared + private async Task Client_UserJoined(SocketGuildUser arg) + { + await UpdateGuildMember(arg); + } + #endregion + private async Task UpdateGuild(SocketGuild g) { try From 74c80c50e942030515ccf4ad14a630fba17833e6 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 8 Dec 2017 13:54:29 -0800 Subject: [PATCH 06/13] Adding notes, possible goals for this branch --- Module/EntityCache/EntityCache.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Module/EntityCache/EntityCache.cs b/Module/EntityCache/EntityCache.cs index 36a66df..352d436 100644 --- a/Module/EntityCache/EntityCache.cs +++ b/Module/EntityCache/EntityCache.cs @@ -16,6 +16,17 @@ namespace Noikoio.RegexBot.Module.EntityCache /// class EntityCache : BotModule { + /* + * Future plans: + * Have this, or something connected to this class, be accessible throughout the bot. + * + * There should be a system that holds a small in-memory cache of users (as EntityCache objects) + * for quick lookups by other parts of the bot. + * Without this system, we'll be having future bot features constantly querying the database + * on their own to look up entity cache records, which (among other things) could result in + * a race conditions where an event becomes aware of a user before it has been recorded. + */ + private readonly DatabaseConfig _db; public override string Name => nameof(EntityCache); From 8bb274bd694f8834abd9598e057f92ba3d13414b Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 22 Dec 2017 17:53:46 -0800 Subject: [PATCH 07/13] Moving EntityCache files to separate directory --- .../EntityCache.cs => EntityCache/Module.cs | 29 ++---- EntityCache/SqlHelper.cs | 98 +++++++++++++++++++ Module/EntityCache/Sql.cs | 56 ----------- RegexBot.cs | 2 +- 4 files changed, 108 insertions(+), 77 deletions(-) rename Module/EntityCache/EntityCache.cs => EntityCache/Module.cs (85%) create mode 100644 EntityCache/SqlHelper.cs delete mode 100644 Module/EntityCache/Sql.cs diff --git a/Module/EntityCache/EntityCache.cs b/EntityCache/Module.cs similarity index 85% rename from Module/EntityCache/EntityCache.cs rename to EntityCache/Module.cs index 352d436..d2cb8dd 100644 --- a/Module/EntityCache/EntityCache.cs +++ b/EntityCache/Module.cs @@ -7,37 +7,26 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -namespace Noikoio.RegexBot.Module.EntityCache +namespace Noikoio.RegexBot.EntityCache { /// - /// Caches information regarding all known guilds, channels, and users. + /// Bot module portion of the entity cache. Caches information regarding all known guilds, channels, and users. /// The function of this module should be transparent to the user, and thus no configuration is needed. /// This module should be initialized BEFORE any other modules that make use of guild and user cache. /// - class EntityCache : BotModule + class Module : BotModule { - /* - * Future plans: - * Have this, or something connected to this class, be accessible throughout the bot. - * - * There should be a system that holds a small in-memory cache of users (as EntityCache objects) - * for quick lookups by other parts of the bot. - * Without this system, we'll be having future bot features constantly querying the database - * on their own to look up entity cache records, which (among other things) could result in - * a race conditions where an event becomes aware of a user before it has been recorded. - */ - private readonly DatabaseConfig _db; public override string Name => nameof(EntityCache); - - public EntityCache(DiscordSocketClient client) : base(client) + + public Module(DiscordSocketClient client) : base(client) { _db = RegexBot.Config.Database; if (_db.Available) { - Sql.CreateCacheTables(); + SqlHelper.CreateCacheTables(); client.GuildAvailable += Client_GuildAvailable; client.GuildUpdated += Client_GuildUpdated; @@ -49,12 +38,12 @@ namespace Noikoio.RegexBot.Module.EntityCache Log("No database storage available.").Wait(); } } - + public override Task ProcessConfiguration(JToken configSection) => Task.FromResult(null); #region Event handling // Guild and guild member information has become available. - // This is a very expensive operation, when joining larger guilds for the first time. + // This is a very expensive operation, especially when joining larger guilds. private async Task Client_GuildAvailable(SocketGuild arg) { await Task.Run(async () => @@ -124,7 +113,7 @@ namespace Noikoio.RegexBot.Module.EntityCache + "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, " + "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time... + "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url"; - + var uid = c.Parameters.Add("@Uid", NpgsqlDbType.Bigint); var gid = c.Parameters.Add("@Gid", NpgsqlDbType.Bigint); c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now; diff --git a/EntityCache/SqlHelper.cs b/EntityCache/SqlHelper.cs new file mode 100644 index 0000000..39d3270 --- /dev/null +++ b/EntityCache/SqlHelper.cs @@ -0,0 +1,98 @@ +using Npgsql; +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +namespace Noikoio.RegexBot.EntityCache +{ + /// + /// Helper methods for database operations. + /// + static class SqlHelper + { + public const string TableGuild = "cache_guild"; + public const string TableTextChannel = "cache_textchannel"; + public const string TableUser = "cache_users"; + + private static async Task OpenDB() + { + if (!RegexBot.Config.Database.Available) return null; + return await RegexBot.Config.Database.GetOpenConnectionAsync(); + } + + public static async Task CreateCacheTables() + { + var db = await OpenDB(); + if (db == null) return; + using (db) + { + // Guild cache + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableGuild + " (" + + "guild_id bigint primary key, " + + "cache_date timestamptz not null, " + + "current_name text not null, " + + "display_name text null" + + ")"; + await c.ExecuteNonQueryAsync(); + } + // May not require other indexes. Add here if they become necessary. + + // Text channel cache + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableTextChannel + " (" + + "channel_id bigint not null primary key, " + + $"guild_id bigint not null references {TableGuild}, " + + "cache_date timestamptz not null, " + + "name text not null"; + await c.ExecuteNonQueryAsync(); + } + // As of the time of this commit, Discord doesn't allow any uppercase characters + // in channel names. No lowercase name index needed. + + // User cache + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableUser + " (" + + "user_id bigint not null, " + + $"guild_id bigint not null references {TableGuild}, " + + "cache_date timestamptz not null, " + + "username text not null, " + + "discriminator text not null, " + + "nickname text null, " + + "avatar_url text null" + + ")"; + await c.ExecuteNonQueryAsync(); + } + using (var c = db.CreateCommand()) + { + // guild_id is a foreign key, and also one half of the primary key here + c.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS " + + $"{TableUser}_ck_idx on {TableUser} (user_id, guild_id)"; + await c.ExecuteNonQueryAsync(); + } + using (var c = db.CreateCommand()) + { + c.CommandText = "CREATE INDEX IF NOT EXISTS " + + $"{TableUser}_usersearch_idx on {TableUser} LOWER(username)"; + await c.ExecuteNonQueryAsync(); + } + } + } + + #region Insertions and updates + static async Task UpdateGuild() + { + var db = await OpenDB(); + if (db == null) return; + using (db) + { + + } + } + #endregion + } +} diff --git a/Module/EntityCache/Sql.cs b/Module/EntityCache/Sql.cs deleted file mode 100644 index ed55b0b..0000000 --- a/Module/EntityCache/Sql.cs +++ /dev/null @@ -1,56 +0,0 @@ -using Npgsql; -using System; -using System.Collections.Generic; -using System.Text; - -namespace Noikoio.RegexBot.Module.EntityCache -{ - /// - /// Contains common constants and static methods for cache access. - /// - class Sql - { - public const string TableGuild = "cache_guild"; - public const string TableUser = "cache_users"; - - private static NpgsqlConnection OpenDB() => - RegexBot.Config.Database.GetOpenConnectionAsync().GetAwaiter().GetResult(); - - public static void CreateCacheTables() - { - using (var db = OpenDB()) - { - using (var c = db.CreateCommand()) - { - c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableGuild + "(" - + "guild_id bigint primary key, " - + "current_name text not null, " - + "display_name text null" - + ")"; - c.ExecuteNonQuery(); - } - - using (var c = db.CreateCommand()) - { - c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableUser + "(" - + "user_id bigint not null, " - + $"guild_id bigint not null references {TableGuild}, " - + "cache_date timestamptz not null, " - + "username text not null, " - + "discriminator text not null, " - + "nickname text null, " - + "avatar_url text null" - + ")"; - c.ExecuteNonQuery(); - } - using (var c = db.CreateCommand()) - { - c.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS " - + $"{TableUser}_idx on {TableUser} (user_id, guild_id)"; - c.ExecuteNonQuery(); - } - // TODO create indexes for string-based queries - } - } - } -} diff --git a/RegexBot.cs b/RegexBot.cs index 97e8b8f..019cead 100644 --- a/RegexBot.cs +++ b/RegexBot.cs @@ -48,7 +48,7 @@ namespace Noikoio.RegexBot new Module.AutoMod.AutoMod(_client), new Module.ModTools.ModTools(_client), new Module.AutoRespond.AutoRespond(_client), - new Module.EntityCache.EntityCache(_client) // EntityCache goes before anything else that uses its data + new EntityCache.Module(_client) // EntityCache goes before anything else that uses its data }; var dlog = Logger.GetLogger("Discord.Net"); _client.Log += async (arg) => From 2e0b4089466aae5b9343364f1ff69fa60f01ec6c Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 22 Dec 2017 19:17:36 -0800 Subject: [PATCH 08/13] Re-adding SQL inserts/updates --- EntityCache/Module.cs | 139 +++++++++++++-------------------------- EntityCache/SqlHelper.cs | 69 +++++++++++++++++-- 2 files changed, 111 insertions(+), 97 deletions(-) diff --git a/EntityCache/Module.cs b/EntityCache/Module.cs index d2cb8dd..2726237 100644 --- a/EntityCache/Module.cs +++ b/EntityCache/Module.cs @@ -2,9 +2,6 @@ using Newtonsoft.Json.Linq; using Noikoio.RegexBot.ConfigItem; using Npgsql; -using NpgsqlTypes; -using System; -using System.Collections.Generic; using System.Threading.Tasks; namespace Noikoio.RegexBot.EntityCache @@ -12,7 +9,7 @@ namespace Noikoio.RegexBot.EntityCache /// /// Bot module portion of the entity cache. Caches information regarding all known guilds, channels, and users. /// The function of this module should be transparent to the user, and thus no configuration is needed. - /// This module should be initialized BEFORE any other modules that make use of guild and user cache. + /// This module should be initialized BEFORE any other modules that make use of the entity cache. /// class Module : BotModule { @@ -26,7 +23,7 @@ namespace Noikoio.RegexBot.EntityCache if (_db.Available) { - SqlHelper.CreateCacheTables(); + SqlHelper.CreateCacheTablesAsync().Wait(); client.GuildAvailable += Client_GuildAvailable; client.GuildUpdated += Client_GuildUpdated; @@ -48,110 +45,66 @@ namespace Noikoio.RegexBot.EntityCache { await Task.Run(async () => { - await UpdateGuild(arg); - await UpdateGuildMember(arg.Users); - } - ); + try + { + await SqlHelper.UpdateGuildAsync(arg); + await SqlHelper.UpdateGuildMemberAsync(arg.Users); + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(Client_GuildAvailable)}: {ex.Message}"); + } + }); } // Guild information has changed private async Task Client_GuildUpdated(SocketGuild arg1, SocketGuild arg2) { - await Task.Run(() => UpdateGuild(arg2)); + await Task.Run(async () => + { + try + { + await SqlHelper.UpdateGuildAsync(arg2); + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(Client_GuildUpdated)}: {ex.Message}"); + } + }); } // Guild member information has changed private async Task Client_GuildMemberUpdated(SocketGuildUser arg1, SocketGuildUser arg2) { - await Task.Run(() => UpdateGuildMember(arg2)); + await Task.Run(async () => + { + try + { + await SqlHelper.UpdateGuildMemberAsync(arg2); + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(Client_GuildMemberUpdated)}: {ex.Message}"); + } + }); } // A new guild member has appeared private async Task Client_UserJoined(SocketGuildUser arg) { - await UpdateGuildMember(arg); + await Task.Run(async () => + { + try + { + await SqlHelper.UpdateGuildMemberAsync(arg); + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(Client_UserJoined)}: {ex.Message}"); + } + }); } #endregion - - private async Task UpdateGuild(SocketGuild g) - { - try - { - using (var db = await _db.GetOpenConnectionAsync()) - { - using (var c = db.CreateCommand()) - { - c.CommandText = "INSERT INTO " + Sql.TableGuild + " (guild_id, current_name) " - + "VALUES (@GuildId, @CurrentName) " - + "ON CONFLICT (guild_id) DO UPDATE SET " - + "current_name = EXCLUDED.current_name"; - c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = g.Id; - c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name; - c.Prepare(); - await c.ExecuteNonQueryAsync(); - } - } - } - catch (NpgsqlException ex) - { - await Log($"SQL error in {nameof(UpdateGuild)}: " + ex.Message); - } - } - - private async Task UpdateGuildMember(IEnumerable users) - { - try - { - using (var db = await _db.GetOpenConnectionAsync()) - { - using (var c = db.CreateCommand()) - { - c.CommandText = "INSERT INTO " + Sql.TableUser - + " (user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url)" - + " VALUES (@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) " - + "ON CONFLICT (user_id, guild_id) DO UPDATE SET " - + "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, " - + "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time... - + "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url"; - - var uid = c.Parameters.Add("@Uid", NpgsqlDbType.Bigint); - var gid = c.Parameters.Add("@Gid", NpgsqlDbType.Bigint); - c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now; - var uname = c.Parameters.Add("@Uname", NpgsqlDbType.Text); - var disc = c.Parameters.Add("@Disc", NpgsqlDbType.Text); - var nname = c.Parameters.Add("@Nname", NpgsqlDbType.Text); - var url = c.Parameters.Add("@Url", NpgsqlDbType.Text); - c.Prepare(); - - foreach (var item in users) - { - if (item.IsBot || item.IsWebhook) continue; - - uid.Value = item.Id; - gid.Value = item.Guild.Id; - uname.Value = item.Username; - disc.Value = item.Discriminator; - nname.Value = item.Nickname; - if (nname.Value == null) nname.Value = DBNull.Value; // why can't ?? work here? - url.Value = item.GetAvatarUrl(); - if (url.Value == null) url.Value = DBNull.Value; - - await c.ExecuteNonQueryAsync(); - } - } - } - } - catch (NpgsqlException ex) - { - await Log($"SQL error in {nameof(UpdateGuildMember)}: " + ex.Message); - } - } - - private Task UpdateGuildMember(SocketGuildUser user) - { - var gid = user.Guild.Id; - var ml = new SocketGuildUser[] { user }; - return UpdateGuildMember(ml); - } + } } diff --git a/EntityCache/SqlHelper.cs b/EntityCache/SqlHelper.cs index 39d3270..c68a8e7 100644 --- a/EntityCache/SqlHelper.cs +++ b/EntityCache/SqlHelper.cs @@ -1,13 +1,15 @@ -using Npgsql; +using Discord.WebSocket; +using Npgsql; +using NpgsqlTypes; using System; using System.Collections.Generic; -using System.Text; using System.Threading.Tasks; namespace Noikoio.RegexBot.EntityCache { /// /// Helper methods for database operations. + /// Exceptions are not handled within methods of this class. /// static class SqlHelper { @@ -21,7 +23,7 @@ namespace Noikoio.RegexBot.EntityCache return await RegexBot.Config.Database.GetOpenConnectionAsync(); } - public static async Task CreateCacheTables() + internal static async Task CreateCacheTablesAsync() { var db = await OpenDB(); if (db == null) return; @@ -84,13 +86,72 @@ namespace Noikoio.RegexBot.EntityCache } #region Insertions and updates - static async Task UpdateGuild() + internal static async Task UpdateGuildAsync(SocketGuild g) { var db = await OpenDB(); if (db == null) return; using (db) { + using (var c = db.CreateCommand()) + { + c.CommandText = "INSERT INTO " + Sql.TableGuild + " (guild_id, current_name) " + + "VALUES (@GuildId, @CurrentName) " + + "ON CONFLICT (guild_id) DO UPDATE SET " + + "current_name = EXCLUDED.current_name"; + c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = g.Id; + c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name; + c.Prepare(); + await c.ExecuteNonQueryAsync(); + } + } + } + internal static Task UpdateGuildMemberAsync(SocketGuildUser user) + { + var ml = new SocketGuildUser[] { user }; + return UpdateGuildMemberAsync(ml); + } + internal static async Task UpdateGuildMemberAsync(IEnumerable users) + { + var db = await OpenDB(); + if (db == null) return; + using (db) + { + using (var c = db.CreateCommand()) + { + c.CommandText = "INSERT INTO " + Sql.TableUser + + " (user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url)" + + " VALUES (@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) " + + "ON CONFLICT (user_id, guild_id) DO UPDATE SET " + + "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, " + + "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time... + + "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url"; + + var uid = c.Parameters.Add("@Uid", NpgsqlDbType.Bigint); + var gid = c.Parameters.Add("@Gid", NpgsqlDbType.Bigint); + c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now; + var uname = c.Parameters.Add("@Uname", NpgsqlDbType.Text); + var disc = c.Parameters.Add("@Disc", NpgsqlDbType.Text); + var nname = c.Parameters.Add("@Nname", NpgsqlDbType.Text); + var url = c.Parameters.Add("@Url", NpgsqlDbType.Text); + c.Prepare(); + + foreach (var item in users) + { + if (item.IsBot || item.IsWebhook) continue; + + uid.Value = item.Id; + gid.Value = item.Guild.Id; + uname.Value = item.Username; + disc.Value = item.Discriminator; + nname.Value = item.Nickname; + if (nname.Value == null) nname.Value = DBNull.Value; // why can't ?? work here? + url.Value = item.GetAvatarUrl(); + if (url.Value == null) url.Value = DBNull.Value; + + await c.ExecuteNonQueryAsync(); + } + } } } #endregion From 1fbe4d5e52c52089f5bb9727435dbbddfc18267b Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 22 Dec 2017 22:20:06 -0800 Subject: [PATCH 09/13] Added EntityCache class with CacheUser Its purpose is documented in the comments within EntityCache.cs. CacheUser has been implemented but is not yet tested. --- .../CacheUser.cs | 135 ++++++++++++------ EntityCache/EntityCache.cs | 38 +++++ EntityCache/SqlHelper.cs | 18 +-- RegexBot.cs | 10 +- 4 files changed, 146 insertions(+), 55 deletions(-) rename Module/EntityCache/UserCacheItem.cs => EntityCache/CacheUser.cs (55%) create mode 100644 EntityCache/EntityCache.cs diff --git a/Module/EntityCache/UserCacheItem.cs b/EntityCache/CacheUser.cs similarity index 55% rename from Module/EntityCache/UserCacheItem.cs rename to EntityCache/CacheUser.cs index 1e94e95..56c58f6 100644 --- a/Module/EntityCache/UserCacheItem.cs +++ b/EntityCache/CacheUser.cs @@ -1,15 +1,18 @@ -using System; +using Discord.WebSocket; +using Npgsql; +using System; using System.Collections.Generic; using System.Data.Common; +using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; -namespace Noikoio.RegexBot.Module.EntityCache +namespace Noikoio.RegexBot.EntityCache { /// - /// Represents a cached user. + /// Representation of a cached user. /// - class UserCacheItem + class CacheUser { readonly ulong _userId; readonly ulong _guildId; @@ -57,8 +60,20 @@ namespace Noikoio.RegexBot.Module.EntityCache /// public string AvatarUrl => _avatarUrl; + private CacheUser(SocketGuildUser u) + { + _userId = u.Id; + _guildId = u.Guild.Id; + _cacheDate = DateTime.UtcNow; + _username = u.Username; + _discriminator = u.Discriminator; + _nickname = u.Nickname; + _avatarUrl = u.GetAvatarUrl(); + } - private UserCacheItem(DbDataReader r) + // Double-check SqlHelper if making changes to this constant + const string QueryColumns = "user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url"; + private CacheUser(DbDataReader r) { // Double-check ordinals if making changes to QueryColumns unchecked @@ -77,20 +92,32 @@ namespace Noikoio.RegexBot.Module.EntityCache public override string ToString() => DisplayName; #region Queries - // Double-check constructor if making changes to this constant - const string QueryColumns = "user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url"; - - /// - /// Attempts to query for an exact result with the given parameters. - /// - /// Null on no result. - public static async Task QueryAsync(ulong guild, ulong user) + // Accessible by EntityCache. Documentation is there. + internal static async Task QueryAsync(DiscordSocketClient c, ulong guild, ulong user) { - using (var db = await RegexBot.Config.Database.GetOpenConnectionAsync()) + // Local cache search + var lresult = LocalQueryAsync(c, guild, user); + if (lresult != null) return lresult; + + // Database cache search + var db = await RegexBot.Config.Database.GetOpenConnectionAsync(); + if (db == null) return null; // Database not available for query. + using (db) return await DbQueryAsync(db, guild, user); + } + + private static CacheUser LocalQueryAsync(DiscordSocketClient c, ulong guild, ulong user) + { + var u = c.GetGuild(guild)?.GetUser(user); + if (u == null) return null; + return new CacheUser(u); + } + private static async Task DbQueryAsync(NpgsqlConnection db, ulong guild, ulong user) + { + using (db) { using (var c = db.CreateCommand()) { - c.CommandText = $"SELECT {QueryColumns} FROM {Sql.TableUser} WHERE " + c.CommandText = $"SELECT {QueryColumns} FROM {SqlHelper.TableUser} WHERE " + "user_id = @Uid AND guild_id = @Gid"; c.Parameters.Add("@Uid", NpgsqlTypes.NpgsqlDbType.Bigint).Value = user; c.Parameters.Add("@Gid", NpgsqlTypes.NpgsqlDbType.Bigint).Value = guild; @@ -99,7 +126,7 @@ namespace Noikoio.RegexBot.Module.EntityCache { if (await r.ReadAsync()) { - return new UserCacheItem(r); + return new CacheUser(r); } else { @@ -110,23 +137,21 @@ namespace Noikoio.RegexBot.Module.EntityCache } } - private static Regex DiscriminatorSearch = new Regex(@"(.+)#(\d{4}(?!\d))", RegexOptions.Compiled); + // ----- - /// - /// Attempts to look up the user given a search string. - /// This string looks up case-insensitive, exact matches of nicknames and usernames. - /// - /// An containing zero or more query results, sorted by cache date. - public static async Task> QueryAsync(ulong guild, string search) + private static Regex DiscriminatorSearch = new Regex(@"(.+)#(\d{4}(?!\d))", RegexOptions.Compiled); + // Accessible by EntityCache. Documentation is there. + internal static async Task> QueryAsync(DiscordSocketClient c, ulong guild, string search) { - // Is search just a number? It's an ID. + // Is search just a number? Assume ID, pass it on to the correct place. if (ulong.TryParse(search, out var presult)) { - var r = await QueryAsync(guild, presult); - if (r == null) return new UserCacheItem[0]; - else return new UserCacheItem[] { r }; + var r = await QueryAsync(c, guild, presult); + if (r == null) return new CacheUser[0]; + else return new CacheUser[] { r }; } + // Split name/discriminator string name; string disc; var split = DiscriminatorSearch.Match(search); @@ -141,14 +166,52 @@ namespace Noikoio.RegexBot.Module.EntityCache disc = null; } - // Storing in HashSet to enforce uniqueness - HashSet result = new HashSet(_uc); + // Local cache search + var lresult = LocalQueryAsync(c, guild, name, disc); + if (lresult.Count() != 0) return lresult; - using (var db = await RegexBot.Config.Database.GetOpenConnectionAsync()) + // Database cache search + var db = await RegexBot.Config.Database.GetOpenConnectionAsync(); + if (db == null) return null; // Database not available for query. + using (db) return await DbQueryAsync(db, guild, name, disc); + } + + private static IEnumerable LocalQueryAsync(DiscordSocketClient c, ulong guild, string name, string disc) + { + var g = c.GetGuild(guild); + if (g == null) return new CacheUser[] { }; + + bool Filter(string iun, string inn, string idc) + { + // Same logic as in the SQL query in the method below this one + bool match = + string.Equals(iun, name, StringComparison.InvariantCultureIgnoreCase) + || string.Equals(inn, name, StringComparison.InvariantCultureIgnoreCase); + + if (match && disc != null) + match = idc.Equals(disc); + + return match; + } + + var qresult = g.Users.Where(i => Filter(i.Username, i.Nickname, i.Discriminator)); + var result = new List(); + foreach (var item in qresult) + { + result.Add(new CacheUser(item)); + } + return result; + } + + private static async Task> DbQueryAsync(NpgsqlConnection db, ulong guild, string name, string disc) + { + var result = new List(); + + using (db = await RegexBot.Config.Database.GetOpenConnectionAsync()) { using (var c = db.CreateCommand()) { - c.CommandText = $"SELECT {QueryColumns} FROM {Sql.TableUser} WHERE " + c.CommandText = $"SELECT {QueryColumns} FROM {SqlHelper.TableUser} WHERE " + "( lower(username) = lower(@NameSearch) OR lower(nickname) = lower(@NameSearch) )"; c.Parameters.Add("@NameSearch", NpgsqlTypes.NpgsqlDbType.Text).Value = name; if (disc != null) @@ -162,21 +225,13 @@ namespace Noikoio.RegexBot.Module.EntityCache { while (await r.ReadAsync()) { - result.Add(new UserCacheItem(r)); + result.Add(new CacheUser(r)); } } } } return result; } - - private static UniqueChecker _uc = new UniqueChecker(); - class UniqueChecker : IEqualityComparer - { - public bool Equals(UserCacheItem x, UserCacheItem y) => x.UserId == y.UserId && x.GuildId == y.GuildId; - - public int GetHashCode(UserCacheItem obj) => unchecked((int)(obj.UserId ^ obj.GuildId)); - } #endregion } } diff --git a/EntityCache/EntityCache.cs b/EntityCache/EntityCache.cs new file mode 100644 index 0000000..be280b1 --- /dev/null +++ b/EntityCache/EntityCache.cs @@ -0,0 +1,38 @@ +using Discord.WebSocket; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Noikoio.RegexBot.EntityCache +{ + /// + /// Static class for accessing the entity cache. + /// + static class EntityCache + { + /* + * The entity cache works by combining data known/cached by Discord.Net in addition to + * what has been stored in the database. If data does not exist in the former, it is + * retrieved from the latter. + * In either case, the resulting data is placed within a cache item object. + */ + + static DiscordSocketClient _client; + internal static void SetClient(DiscordSocketClient c) => _client = _client ?? c; + + /// + /// Attempts to query for an exact result with the given parameters. + /// Does not handle exceptions that may occur. + /// + /// Null on no result. + internal static Task QueryAsync(ulong guild, ulong user) + => CacheUser.QueryAsync(_client, guild, user); + + /// + /// Attempts to look up the user given a search string. + /// This string looks up case-insensitive, exact matches of nicknames and usernames. + /// + /// An containing zero or more query results, sorted by cache date. + internal static Task> QueryAsync(DiscordSocketClient c, ulong guild, string search) + => CacheUser.QueryAsync(_client, guild, search); + } +} diff --git a/EntityCache/SqlHelper.cs b/EntityCache/SqlHelper.cs index c68a8e7..3c8b6ce 100644 --- a/EntityCache/SqlHelper.cs +++ b/EntityCache/SqlHelper.cs @@ -1,5 +1,4 @@ using Discord.WebSocket; -using Npgsql; using NpgsqlTypes; using System; using System.Collections.Generic; @@ -17,15 +16,10 @@ namespace Noikoio.RegexBot.EntityCache public const string TableTextChannel = "cache_textchannel"; public const string TableUser = "cache_users"; - private static async Task OpenDB() - { - if (!RegexBot.Config.Database.Available) return null; - return await RegexBot.Config.Database.GetOpenConnectionAsync(); - } - + // Reminder: Check Cache query methods if making changes to tables internal static async Task CreateCacheTablesAsync() { - var db = await OpenDB(); + var db = await RegexBot.Config.Database.GetOpenConnectionAsync(); if (db == null) return; using (db) { @@ -88,13 +82,13 @@ namespace Noikoio.RegexBot.EntityCache #region Insertions and updates internal static async Task UpdateGuildAsync(SocketGuild g) { - var db = await OpenDB(); + var db = await RegexBot.Config.Database.GetOpenConnectionAsync(); if (db == null) return; using (db) { using (var c = db.CreateCommand()) { - c.CommandText = "INSERT INTO " + Sql.TableGuild + " (guild_id, current_name) " + c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) " + "VALUES (@GuildId, @CurrentName) " + "ON CONFLICT (guild_id) DO UPDATE SET " + "current_name = EXCLUDED.current_name"; @@ -113,13 +107,13 @@ namespace Noikoio.RegexBot.EntityCache } internal static async Task UpdateGuildMemberAsync(IEnumerable users) { - var db = await OpenDB(); + var db = await RegexBot.Config.Database.GetOpenConnectionAsync(); if (db == null) return; using (db) { using (var c = db.CreateCommand()) { - c.CommandText = "INSERT INTO " + Sql.TableUser + c.CommandText = "INSERT INTO " + TableUser + " (user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url)" + " VALUES (@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) " + "ON CONFLICT (user_id, guild_id) DO UPDATE SET " diff --git a/RegexBot.cs b/RegexBot.cs index 019cead..b69db23 100644 --- a/RegexBot.cs +++ b/RegexBot.cs @@ -20,7 +20,7 @@ namespace Noikoio.RegexBot internal RegexBot() { - // Load configuration + // Load initial configuration _config = new Configuration(this); if (!_config.LoadInitialConfig()) { @@ -31,6 +31,7 @@ namespace Noikoio.RegexBot Environment.Exit(1); } + // Set Discord client settings _client = new DiscordSocketClient(new DiscordSocketConfig() { LogLevel = LogSeverity.Info, @@ -39,8 +40,9 @@ namespace Noikoio.RegexBot MessageCacheSize = 0 }); - // Hook up handlers for basic functions + // Hook up basic handlers and other references _client.Connected += _client_Connected; + EntityCache.EntityCache.SetClient(_client); // Initialize modules _modules = new BotModule[] @@ -50,13 +52,15 @@ namespace Noikoio.RegexBot new Module.AutoRespond.AutoRespond(_client), new EntityCache.Module(_client) // EntityCache goes before anything else that uses its data }; + + // Set up logging var dlog = Logger.GetLogger("Discord.Net"); _client.Log += async (arg) => await dlog( String.Format("{0}: {1}{2}", arg.Source, ((int)arg.Severity < 3 ? arg.Severity + ": " : ""), arg.Message)); - // With modules initialized, finish loading configuration + // Finish loading configuration var conf = _config.ReloadServerConfig().Result; if (conf == false) { From b70f57f0a34d7f335f8ae0c78ade54d04b7fece7 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Fri, 22 Dec 2017 22:28:58 -0800 Subject: [PATCH 10/13] Added another handler --- EntityCache/Module.cs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/EntityCache/Module.cs b/EntityCache/Module.cs index 2726237..f976632 100644 --- a/EntityCache/Module.cs +++ b/EntityCache/Module.cs @@ -29,6 +29,7 @@ namespace Noikoio.RegexBot.EntityCache client.GuildUpdated += Client_GuildUpdated; client.GuildMemberUpdated += Client_GuildMemberUpdated; client.UserJoined += Client_UserJoined; + client.UserLeft += Client_UserLeft; } else { @@ -37,8 +38,7 @@ namespace Noikoio.RegexBot.EntityCache } public override Task ProcessConfiguration(JToken configSection) => Task.FromResult(null); - - #region Event handling + // Guild and guild member information has become available. // This is a very expensive operation, especially when joining larger guilds. private async Task Client_GuildAvailable(SocketGuild arg) @@ -104,7 +104,21 @@ namespace Noikoio.RegexBot.EntityCache } }); } - #endregion - + + // User left the guild. No new data, but gives an excuse to update the cache date. + private async Task Client_UserLeft(SocketGuildUser arg) + { + await Task.Run(async () => + { + try + { + await SqlHelper.UpdateGuildMemberAsync(arg); + } + catch (NpgsqlException ex) + { + await Log($"SQL error in {nameof(Client_UserLeft)}: {ex.Message}"); + } + }); + } } } From 71ce62e88cf53dd1e9087ba71d177120c48171b9 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Thu, 4 Jan 2018 14:05:10 -0800 Subject: [PATCH 11/13] Fix incorrect method signature --- EntityCache/EntityCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EntityCache/EntityCache.cs b/EntityCache/EntityCache.cs index be280b1..5937f40 100644 --- a/EntityCache/EntityCache.cs +++ b/EntityCache/EntityCache.cs @@ -32,7 +32,7 @@ namespace Noikoio.RegexBot.EntityCache /// This string looks up case-insensitive, exact matches of nicknames and usernames. /// /// An containing zero or more query results, sorted by cache date. - internal static Task> QueryAsync(DiscordSocketClient c, ulong guild, string search) + internal static Task> QueryAsync(ulong guild, string search) => CacheUser.QueryAsync(_client, guild, search); } } From 11fa202afd16a60773140b6f62160030de57b34d Mon Sep 17 00:00:00 2001 From: Noikoio Date: Thu, 4 Jan 2018 14:05:42 -0800 Subject: [PATCH 12/13] Debug profile was not working until now --- Properties/PublishProfiles/DebugProfile.pubxml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Properties/PublishProfiles/DebugProfile.pubxml b/Properties/PublishProfiles/DebugProfile.pubxml index e2bc34c..a6217fd 100644 --- a/Properties/PublishProfiles/DebugProfile.pubxml +++ b/Properties/PublishProfiles/DebugProfile.pubxml @@ -1,13 +1,13 @@  FileSystem Debug - netcoreapp1.1 + netcoreapp2.0 bin\Debug\PublishOutput \ No newline at end of file From 74ac34fcf1c6982b7794c8116dffdf8d5fac0dc2 Mon Sep 17 00:00:00 2001 From: Noikoio Date: Thu, 4 Jan 2018 14:06:04 -0800 Subject: [PATCH 13/13] Fixed table setup, updates --- EntityCache/SqlHelper.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/EntityCache/SqlHelper.cs b/EntityCache/SqlHelper.cs index 3c8b6ce..d19489c 100644 --- a/EntityCache/SqlHelper.cs +++ b/EntityCache/SqlHelper.cs @@ -43,7 +43,8 @@ namespace Noikoio.RegexBot.EntityCache + "channel_id bigint not null primary key, " + $"guild_id bigint not null references {TableGuild}, " + "cache_date timestamptz not null, " - + "name text not null"; + + "channel_name text not null" + + ")"; await c.ExecuteNonQueryAsync(); } // As of the time of this commit, Discord doesn't allow any uppercase characters @@ -73,7 +74,7 @@ namespace Noikoio.RegexBot.EntityCache using (var c = db.CreateCommand()) { c.CommandText = "CREATE INDEX IF NOT EXISTS " - + $"{TableUser}_usersearch_idx on {TableUser} LOWER(username)"; + + $"{TableUser}_usersearch_idx on {TableUser} (LOWER(username))"; await c.ExecuteNonQueryAsync(); } } @@ -88,12 +89,13 @@ namespace Noikoio.RegexBot.EntityCache { using (var c = db.CreateCommand()) { - c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) " - + "VALUES (@GuildId, @CurrentName) " + c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, cache_date, current_name) " + + "VALUES (@GuildId, @Date, @CurrentName) " + "ON CONFLICT (guild_id) DO UPDATE SET " - + "current_name = EXCLUDED.current_name"; + + "current_name = EXCLUDED.current_name, cache_date = EXCLUDED.cache_date"; c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = g.Id; c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name; + c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now; c.Prepare(); await c.ExecuteNonQueryAsync(); }