Noi/RegexBot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Finished implementing database caches

Browse source
This commit is contained in:
Noikoio 2017-11-05 20:55:57 -08:00
parent 2958d14b08
commit b67716ec94
4 changed files with 142 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
BotFeature.cs
View file

@ -20,7 +20,7 @@ namespace Noikoio.RegexBot
public abstract string Name { get; }
protected DiscordSocketClient Client => _client;
protected BotFeature(DiscordSocketClient client)
public BotFeature(DiscordSocketClient client)
{
_client = client;
_logger = Logger.GetLogger(this.Name);

140
Feature/DBCache/EntityCache.cs
View file

@ -1,6 +1,7 @@
using Discord.WebSocket;
using Newtonsoft.Json.Linq;
using Noikoio.RegexBot.ConfigItem;
using Npgsql;
using NpgsqlTypes;
using System;
using System.Collections.Generic;
@ -24,6 +25,8 @@ namespace Noikoio.RegexBot.Feature.DBCache
if (_db.Enabled)
{
CreateCacheTables();
client.GuildAvailable += Client_GuildAvailable;
client.GuildUpdated += Client_GuildUpdated;
client.GuildMemberUpdated += Client_GuildMemberUpdated;
@ -41,10 +44,12 @@ namespace Noikoio.RegexBot.Feature.DBCache
// Guild _and_ guild member information has become available
private async Task Client_GuildAvailable(SocketGuild arg)
{
await CreateCacheTables(arg.Id);
await Task.Run(() => UpdateGuild(arg));
await Task.Run(() => UpdateGuildMember(arg.Id, arg.Users));
await Task.Run(async () =>
{
await UpdateGuild(arg);
await UpdateGuildMember(arg.Users);
}
);
}
// Guild information has changed
@ -58,15 +63,15 @@ namespace Noikoio.RegexBot.Feature.DBCache
{
await Task.Run(() => UpdateGuildMember(arg2));
}
#endregion
#endregion
#region Table setup
#region Table setup
public const string TableGuild = "cache_guild";
const string TableUser = "cache_users";
public const string TableUser = "cache_users";
private async Task CreateCacheTables(ulong gid)
private void CreateCacheTables()
{
using (var db = await _db.GetOpenConnectionAsync())
using (var db = _db.GetOpenConnectionAsync().GetAwaiter().GetResult())
{
using (var c = db.CreateCommand())
{
@ -75,89 +80,110 @@ namespace Noikoio.RegexBot.Feature.DBCache
+ "current_name text not null, "
+ "display_name text null"
+ ")";
// TODO determine if other columns necessary?
await c.ExecuteNonQueryAsync();
// TODO determine if other columns might be needed?
c.ExecuteNonQuery();
}
using (var c = db.CreateCommand())
{
c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableUser + "("
+ "user_id bigint, "
+ "guild_id bigint references " + TableGuild + " (guild_id), "
+ "user_id bigint not null, "
+ $"guild_id bigint not null references {TableGuild}, "
+ "cache_date timestamptz not null, "
+ "username text not null, "
+ "discriminator text not null, "
+ "nickname text null, "
+ "avatar_url text null"
+ ")";
await c.ExecuteNonQueryAsync();
c.ExecuteNonQuery();
}
using (var c = db.CreateCommand())
{
c.CommandText = "CREATE UNIQUE INDEX IF NOT EXISTS "
+ $"{TableUser}_idx on {TableUser} (user_id, guild_id)";
await c.ExecuteNonQueryAsync();
c.ExecuteNonQuery();
}
}
}
#endregion
#endregion
private async Task UpdateGuild(SocketGuild g)
{
using (var db = await _db.GetOpenConnectionAsync())
try
{
using (var c = db.CreateCommand())
using (var db = await _db.GetOpenConnectionAsync())
{
c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) "
+ "(@GuildId, @CurrentName) "
+ "ON CONFLICT (guild_id) DO UPDATE SET "
+ "current_name = EXCLUDED.current_name";
c.Parameters.Add("@GuildID", NpgsqlDbType.Bigint).Value = g.Id;
c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name;
c.Prepare();
await c.ExecuteNonQueryAsync();
}
}
}
private async Task UpdateGuildMember(ulong gid, IEnumerable<SocketGuildUser> users)
{
using (var db = await _db.GetOpenConnectionAsync())
{
using (var c = db.CreateCommand())
{
c.CommandText = "INSERT INTO " + TableUser + " VALUES "
+ "(@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) "
+ "ON CONFLICT (user_id, guild_id) DO UPDATE SET "
+ "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, "
+ "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time...
+ "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url";
c.Prepare();
var now = DateTime.Now;
List<Task> inserts = new List<Task>();
foreach (var item in users)
using (var c = db.CreateCommand())
{
c.Parameters.Clear();
c.Parameters.Add("@Uid", NpgsqlDbType.Bigint).Value = item.Id;
c.Parameters.Add("@Gid", NpgsqlDbType.Bigint).Value = item.Guild.Id;
c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = now;
c.Parameters.Add("@Uname", NpgsqlDbType.Text).Value = item.Username;
c.Parameters.Add("@Disc", NpgsqlDbType.Text).Value = item.Discriminator;
c.Parameters.Add("@Nname", NpgsqlDbType.Text).Value = item.Nickname;
c.Parameters.Add("@Url", NpgsqlDbType.Text).Value = item.GetAvatarUrl();
c.CommandText = "INSERT INTO " + TableGuild + " (guild_id, current_name) "
+ "VALUES (@GuildId, @CurrentName) "
+ "ON CONFLICT (guild_id) DO UPDATE SET "
+ "current_name = EXCLUDED.current_name";
c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = g.Id;
c.Parameters.Add("@CurrentName", NpgsqlDbType.Text).Value = g.Name;
c.Prepare();
await c.ExecuteNonQueryAsync();
}
}
}
catch (NpgsqlException ex)
{
await Log($"SQL error in {nameof(UpdateGuild)}: " + ex.Message);
}
}
private async Task UpdateGuildMember(IEnumerable<SocketGuildUser> users)
{
try
{
using (var db = await _db.GetOpenConnectionAsync())
{
using (var c = db.CreateCommand())
{
c.CommandText = "INSERT INTO " + TableUser
+ " (user_id, guild_id, cache_date, username, discriminator, nickname, avatar_url)"
+ " VALUES (@Uid, @Gid, @Date, @Uname, @Disc, @Nname, @Url) "
+ "ON CONFLICT (user_id, guild_id) DO UPDATE SET "
+ "cache_date = EXCLUDED.cache_date, username = EXCLUDED.username, "
+ "discriminator = EXCLUDED.discriminator, " // I've seen someone's discriminator change this one time...
+ "nickname = EXCLUDED.nickname, avatar_url = EXCLUDED.avatar_url";
var uid = c.Parameters.Add("@Uid", NpgsqlDbType.Bigint);
var gid = c.Parameters.Add("@Gid", NpgsqlDbType.Bigint);
c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = DateTime.Now;
var uname = c.Parameters.Add("@Uname", NpgsqlDbType.Text);
var disc = c.Parameters.Add("@Disc", NpgsqlDbType.Text);
var nname = c.Parameters.Add("@Nname", NpgsqlDbType.Text);
var url = c.Parameters.Add("@Url", NpgsqlDbType.Text);
c.Prepare();
foreach (var item in users)
{
uid.Value = item.Id;
gid.Value = item.Guild.Id;
uname.Value = item.Username;
disc.Value = item.Discriminator;
nname.Value = item.Nickname;
if (nname.Value == null) nname.Value = DBNull.Value; // why can't ?? work here?
url.Value = item.GetAvatarUrl();
if (url.Value == null) url.Value = DBNull.Value;
await c.ExecuteNonQueryAsync();
}
}
}
}
catch (NpgsqlException ex)
{
await Log($"SQL error in {nameof(UpdateGuildMember)}: " + ex.Message);
}
}
private Task UpdateGuildMember(SocketGuildUser user)
{
var gid = user.Guild.Id;
var ml = new SocketGuildUser[] { user };
return UpdateGuildMember(gid, ml);
return UpdateGuildMember(ml);
}
}
}

92
Feature/DBCache/MessageCache.cs
View file

@ -1,12 +1,11 @@
using Discord.WebSocket;
using Newtonsoft.Json.Linq;
using Noikoio.RegexBot.ConfigItem;
using System;
using System.Collections.Generic;
using System.Text;
using Npgsql;
using NpgsqlTypes;
using System.Threading.Tasks;
namespace Noikoio.RegexBot.Feature.DBCaches
namespace Noikoio.RegexBot.Feature.DBCache
{
/// <summary>
/// Caches information regarding all incoming messages.
@ -14,31 +13,38 @@ namespace Noikoio.RegexBot.Feature.DBCaches
/// </summary>
class MessageCache : BotFeature
{
// TODO Something that clears expired cache items
private readonly DatabaseConfig _db;
public override string Name => nameof(MessageCache);
#region Table setup
const string TableGuild = "cache_guild";
const string TableUser = "cache_users";
const string TableMessage = "cache_messages";
public MessageCache(DiscordSocketClient client) : base(client)
{
_db = RegexBot.Config.Database;
client.MessageReceived += Client_MessageReceived;
//client.MessageUpdated += Client_MessageUpdated;
if (_db.Enabled)
{
CreateCacheTables();
client.MessageReceived += Client_MessageReceived;
//client.MessageUpdated += Client_MessageUpdated;
}
else
{
Log("No database storage available.").Wait();
}
}
#region Table setup
const string TableMessage = "cache_messages";
public override Task<object> ProcessConfiguration(JToken configSection) => Task.FromResult<object>(null);
#region Event handling
// A new message has been created
private async Task Client_MessageReceived(SocketMessage arg)
{
if (!_db.Enabled) return;
throw new NotImplementedException();
await Task.Run(() => CacheMessage(arg));
}
//private Task Client_MessageUpdated(Discord.Cacheable<Discord.IMessage, ulong> arg1, SocketMessage arg2, ISocketMessageChannel arg3)
@ -50,28 +56,24 @@ namespace Noikoio.RegexBot.Feature.DBCaches
*/
#endregion
private async Task CreateCacheTables(ulong gid)
private void CreateCacheTables()
{
/* Note:
* We save information per guild in their own schemas named "g_NUM", where NUM is the Guild ID.
*
* The creation of these schemas is handled within here, but we're possibly facing a short delay
* in the event that other events that we're listening for come in without a schema having been
* created yet in which to put them in.
* Got to figure that out.
*/
await _db.CreateGuildSchemaAsync(gid);
using (var db = await _db.OpenConnectionAsync(gid))
using (var db = _db.GetOpenConnectionAsync().GetAwaiter().GetResult())
{
using (var c = db.CreateCommand())
{
c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableMessage + "("
+ "snowflake bigint primary key, "
+ "cache_date timestamptz not null, "
+ "author bigint not null"
c.CommandText = "CREATE TABLE IF NOT EXISTS " + TableMessage + " ("
+ "message_id bigint primary key, "
+ "author_id bigint not null, "
+ "guild_id bigint not null, "
+ "channel_id bigint not null, " // channel cache later? something to think about...
+ "created_ts timestamptz not null, "
+ "edited_ts timestamptz null, "
+ "message text not null, "
+ $"FOREIGN KEY (author_id, guild_id) references {EntityCache.TableUser} (user_id, guild_id)"
+ ")";
await c.ExecuteNonQueryAsync();
// TODO figure out how to store message edits
c.ExecuteNonQuery();
}
}
}
@ -79,12 +81,30 @@ namespace Noikoio.RegexBot.Feature.DBCaches
private async Task CacheMessage(SocketMessage msg)
{
throw new NotImplementedException();
}
private async Task UpdateMessage(SocketMessage msg)
{
throw new NotImplementedException();
try
{
using (var db = await _db.GetOpenConnectionAsync())
{
using (var c = db.CreateCommand())
{
c.CommandText = "INSERT INTO " + TableMessage
+ " (message_id, author_id, guild_id, channel_id, created_ts, message) VALUES "
+ "(@MessageId, @UserId, @GuildId, @ChannelId, @Date, @Message)";
c.Parameters.Add("@MessageId", NpgsqlDbType.Bigint).Value = msg.Id;
c.Parameters.Add("@UserId", NpgsqlDbType.Bigint).Value = msg.Author.Id;
c.Parameters.Add("@GuildId", NpgsqlDbType.Bigint).Value = ((SocketGuildUser)msg.Author).Guild.Id;
c.Parameters.Add("@ChannelId", NpgsqlDbType.Bigint).Value = msg.Channel.Id;
c.Parameters.Add("@Date", NpgsqlDbType.TimestampTZ).Value = msg.Timestamp;
c.Parameters.Add("@Message", NpgsqlDbType.Text).Value = msg.Content;
c.Prepare();
await c.ExecuteNonQueryAsync();
}
}
}
catch (NpgsqlException ex)
{
await Log($"SQL error in {nameof(CacheMessage)}: " + ex.Message);
}
}
}
}

4
RegexBot.cs
View file

@ -36,7 +36,7 @@ namespace Noikoio.RegexBot
LogLevel = LogSeverity.Info,
AlwaysDownloadUsers = true,
DefaultRetryMode = RetryMode.AlwaysRetry,
MessageCacheSize = 50
MessageCacheSize = 0
});
// Hook up handlers for basic functions
@ -48,7 +48,7 @@ namespace Noikoio.RegexBot
new Feature.AutoMod.AutoMod(_client),
new Feature.ModTools.ModTools(_client),
new Feature.AutoRespond.AutoRespond(_client),
new Feature.DBCache.EntityCache(_client),
new Feature.DBCache.EntityCache(_client), // EntityCache goes before anything else that uses its data
new Feature.DBCache.MessageCache(_client)
};
var dlog = Logger.GetLogger("Discord.Net");