From b7567e6b28925524c00beda5e29fa786bea2bf7a Mon Sep 17 00:00:00 2001
From: sabihoshi <sabihoshi.dev@gmail.com>
Date: Fri, 14 Oct 2022 18:58:31 +0800
Subject: [PATCH] feat: Culling

---
 HuTao.Culling/HuTao.Culling.csproj | 16 ++++++
 HuTao.Culling/Program.cs           | 84 ++++++++++++++++++++++++++++++
 HuTao.sln                          |  6 +++
 3 files changed, 106 insertions(+)
 create mode 100644 HuTao.Culling/HuTao.Culling.csproj
 create mode 100644 HuTao.Culling/Program.cs

diff --git a/HuTao.Culling/HuTao.Culling.csproj b/HuTao.Culling/HuTao.Culling.csproj
new file mode 100644
index 0000000..ae72231
--- /dev/null
+++ b/HuTao.Culling/HuTao.Culling.csproj
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\HuTao.Bot\HuTao.Bot.csproj" />
+      <ProjectReference Include="..\HuTao.Data\HuTao.Data.csproj" />
+      <ProjectReference Include="..\HuTao.Services\HuTao.Services.csproj" />
+    </ItemGroup>
+
+</Project>
diff --git a/HuTao.Culling/Program.cs b/HuTao.Culling/Program.cs
new file mode 100644
index 0000000..541fa1e
--- /dev/null
+++ b/HuTao.Culling/Program.cs
@@ -0,0 +1,84 @@
+using HuTao.Data;
+using HuTao.Data.Config;
+using HuTao.Data.Models.Logging;
+using HuTao.Services.Linking;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Serilog;
+
+namespace HuTao.Culling;
+
+public class Program
+{
+    private static IServiceProvider? _services;
+    private HuTaoContext? _db;
+
+    public async Task CullMessagesInternalAsync(DateTimeOffset oldest)
+    {
+        var messageSet = _db!.Set<MessageLog>();
+        Log.Information("Getting messages to delete");
+        var messages = await messageSet
+            .Include(m => m.Embeds)
+            .Include(m => m.Components)
+            .Include(m => m.Attachments)
+            .Where(m => oldest > m.LogDate)
+            .Take(10000)
+            .OrderBy(m => m.LogDate)
+            .ToListAsync();
+        Log.Information("Culling messages older than {Oldest}, total {Count}", oldest, messages.Count);
+
+        var count = 0;
+        foreach (var message in messages)
+        {
+            if (count % 1000 == 0)
+                Log.Information("Deleting message {MessageId}", message.MessageId);
+
+            if (count % 10000 == 0)
+            {
+                Log.Information("Saving changes...");
+                await _db.SaveChangesAsync();
+                Log.Information("Finished saving");
+                Log.Information("Deleted {Count} messages so far...", count);
+            }
+
+            Remove(message);
+        }
+
+        Log.Information("Deleted {Count} messages", count);
+        Log.Information("Final saving changes...");
+        await _db.SaveChangesAsync();
+        Log.Information("Deleted {Count} messages", count);
+
+        void Remove(MessageLog message)
+        {
+            _db.TryRemove(message.Embeds);
+            _db.TryRemove(message.Components);
+            _db.RemoveRange(message.Attachments);
+            _db.Remove(message);
+
+            count++;
+        }
+    }
+
+    public static Task Main(string[] args) => new Program().StartAsync(args);
+
+    public async Task StartAsync(string[] args)
+    {
+        _services = ConfigureServices();
+        _db       = _services.GetRequiredService<HuTaoContext>();
+
+        var days = int.Parse(args.First());
+        var oldest = DateTimeOffset.Now.ToUniversalTime() - TimeSpan.FromDays(days);
+        await CullMessagesInternalAsync(oldest);
+    }
+
+    private static ServiceProvider ConfigureServices() =>
+        new ServiceCollection()
+            .AddLogging(l => l.AddSerilog())
+            .AddDbContext<HuTaoContext>(ContextOptions)
+            .BuildServiceProvider();
+
+    private static void ContextOptions(DbContextOptionsBuilder optionsBuilder) => optionsBuilder
+        .UseLazyLoadingProxies()
+        .UseNpgsql(HuTaoConfig.Configuration.HuTaoContext);
+}
\ No newline at end of file
diff --git a/HuTao.sln b/HuTao.sln
index 10f31be..0c51a2a 100644
--- a/HuTao.sln
+++ b/HuTao.sln
@@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HuTao.Services", "HuTao.Ser
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HuTao.Data", "HuTao.Data\HuTao.Data.csproj", "{DE501463-C094-4550-82F3-9513C576E5B2}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HuTao.Culling", "HuTao.Culling\HuTao.Culling.csproj", "{CD984E6D-8385-441C-A37B-33F2FFD0BFBD}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -27,6 +29,10 @@ Global
 		{DE501463-C094-4550-82F3-9513C576E5B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{DE501463-C094-4550-82F3-9513C576E5B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{DE501463-C094-4550-82F3-9513C576E5B2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CD984E6D-8385-441C-A37B-33F2FFD0BFBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CD984E6D-8385-441C-A37B-33F2FFD0BFBD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CD984E6D-8385-441C-A37B-33F2FFD0BFBD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CD984E6D-8385-441C-A37B-33F2FFD0BFBD}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE