Merge pull request #1 from nationalarchives/feature/DPS-978_upgrade_f…

…or_OpenSearch_215 Feature/dps 978 upgrade for open search 215
nationalarchives · Jan 10, 2025 · 297be31 · 297be31
2 parents 6d4594a + ccd200e
commit 297be31
Show file tree

Hide file tree

Showing 91 changed files with 1,786 additions and 1,225 deletions.
diff --git a/.gitignore b/.gitignore
@@ -268,3 +268,5 @@ aws-logger-errors.txt
 /Dockerfile_web
 /Dockerfile_api
 /run
+/NationalArchives.Taxonomy.Batch.Update.Elastic/Properties/launchSettings.json
+/tna.taxonomy.api/Properties/launchSettings.json
diff --git a/NationalArchives.ActiveMQ/NationalArchives.ActiveMQ.csproj b/NationalArchives.ActiveMQ/NationalArchives.ActiveMQ.csproj
@@ -18,8 +18,8 @@ v1.4: Compilation changes from x64 to Any CPU for compatibiity with Elastic Sear
 v1.5: Updated Nuget Packages
 v1.6 Reverted Apache.NMS and Apache.NMS.ActiveMQ from v2.0 to 1.8 due to assembly ;pad/resolution/impelementation issues.</PackageReleaseNotes>
     <RepositoryUrl>https://tna.visualstudio.com/Discovery/_versionControl?path=$/Discovery/Projects/ActiveMQ</RepositoryUrl>
-    <PackageTags>ActiveMQ queue  elastic taxonomy</PackageTags>
-    <Copyright>"Copyright © The National Archives 2021"</Copyright>
+    <PackageTags>ActiveMQ queue Open Search taxonomy</PackageTags>
+    <Copyright>"Copyright © The National Archives 2024"</Copyright>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <NeutralLanguage></NeutralLanguage>
     <AssemblyVersion>1.6.0.0</AssemblyVersion>

diff --git a/NationalArchives.Taxonomy.Batch.UnitTests/DailyUpdateTest.cs b/NationalArchives.Taxonomy.Batch.UnitTests/DailyUpdateTest.cs
@@ -21,7 +21,7 @@ public class UnitTest1
 
         ISourceIaidInputQueueConsumer _iaidConsumer = Substitute.For<ISourceIaidInputQueueConsumer>();
 
-        IUpdateElasticService _updateElasticService = Substitute.For<IUpdateElasticService>();
+        IUpdateOpenSearchService _updateOpenSearchService = Substitute.For<IUpdateOpenSearchService>();
 
 
         //[TestMethod]

diff --git a/NationalArchives.Taxonomy.Batch.UnitTests/NationalArchives.Taxonomy.Batch.UnitTests.csproj b/NationalArchives.Taxonomy.Batch.UnitTests/NationalArchives.Taxonomy.Batch.UnitTests.csproj
@@ -1,15 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
-    <PackageReference Include="MSTest.TestAdapter" Version="2.2.8" />
-    <PackageReference Include="MSTest.TestFramework" Version="2.2.7" />
-    <PackageReference Include="NSubstitute" Version="4.3.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="MSTest.TestAdapter" Version="3.6.3" />
+    <PackageReference Include="MSTest.TestFramework" Version="3.6.3" />
+    <PackageReference Include="NSubstitute" Version="5.3.0" />
+    <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
+    <PackageReference Include="System.Reflection.Metadata" Version="8.0.1" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/NationalArchives.Taxonomy.Batch.Update.Elastic/ElasticUpdateParams.cs b/NationalArchives.Taxonomy.Batch.Update.Elastic/ElasticUpdateParams.cs
diff --git a/...ives.Taxonomy.Batch.Update.Elastic.csproj → ...s.Taxonomy.Batch.Update.OpenSearch.csproj b/...ives.Taxonomy.Batch.Update.Elastic.csproj → ...s.Taxonomy.Batch.Update.OpenSearch.csproj
@@ -2,19 +2,21 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <IsTransformWebConfigDisabled>true</IsTransformWebConfigDisabled>
+    <UserSecretsId>0acc472f-079b-40ce-8c52-b94d0f5becba</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.AspNetCore.Hosting.WindowsServices" Version="6.0.10" />
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="6.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.1" />
-    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="6.0.0" />
-    <PackageReference Include="NLog.Extensions.Logging" Version="5.0.4" />
+    <PackageReference Include="Microsoft.AspNetCore.Hosting.WindowsServices" Version="8.0.11" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging.EventLog" Version="8.0.1" />
+    <PackageReference Include="NationalArchives.ActiveMQ" Version="1.7.0" />
+    <PackageReference Include="NLog.Extensions.Logging" Version="5.3.14" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/NationalArchives.Taxonomy.Batch.Update.Elastic/OpenSearchUpdateParams.cs b/NationalArchives.Taxonomy.Batch.Update.Elastic/OpenSearchUpdateParams.cs
@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace NationalArchives.Taxonomy.Batch
+{
+    internal sealed class OpenSearchUpdateParams
+    {
+        public int  BulkUpdateBatchSize { get; set; }
+
+        public int QueueFetchSleepTime { get; set; }
+
+        public int SearchDatabaseUpdateInterval { get; set; }
+
+        public int MaxInternalQueueSize { get; set; }
+    }
+}
diff --git a/NationalArchives.Taxonomy.Batch.Update.Elastic/Program.cs b/NationalArchives.Taxonomy.Batch.Update.Elastic/Program.cs
@@ -1,25 +1,22 @@
-using AutoMapper;
-using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.EventLog;
-using NationalArchives.Taxonomy.Batch.Update.Elastic.Service;
+using NationalArchives.Taxonomy.Batch.Update.OpenSearch.Service;
 using NationalArchives.Taxonomy.Common.Domain.Queue;
-using NationalArchives.Taxonomy.Common.Domain.Repository.Common;
-using NationalArchives.Taxonomy.Common.Domain.Repository.Elastic;
-using NationalArchives.Taxonomy.Common.Domain.Repository.Mongo;
+using NationalArchives.Taxonomy.Common.Domain.Repository.OpenSearch;
 using NationalArchives.Taxonomy.Common.Service.Impl;
 using NationalArchives.Taxonomy.Common.Service.Interface;
 using NLog.Extensions.Logging;
 using System;
 using System.Text;
 
-namespace NationalArchives.Taxonomy.Batch.Update.Elastic
+namespace NationalArchives.Taxonomy.Batch.Update.OpenSearch
 {
     class Program
     {
-        private const string EVENT_SOURCE = "Taxonomy Elastic Search Update";
+        private const string EVENT_SOURCE = "Taxonomy Open Search Update";
 
         public static void Main(string[] args)
         {
@@ -33,14 +30,14 @@ public static void Main(string[] args)
                 using (var loggerFactory = LoggerFactory.Create(builder => builder.AddConsole().AddDebug().AddEventSourceLogger().AddEventLog(eventLogSettings)))
                 {
                     serviceLogger = loggerFactory.CreateLogger<Program>();
-                    serviceLogger.LogInformation("Starting the taxonomy elastic update service.");
+                    serviceLogger.LogInformation("Starting the taxonomy Open Search update service.");
                 }
 
                 CreateHostBuilder(args).Build().Run();
             }
             catch (Exception e)
             {
-                StringBuilder sb = new StringBuilder("An error occurred whilst initialising or running the taxonomy elastic search update:");
+                StringBuilder sb = new StringBuilder("An error occurred whilst initialising or running the taxonomy Open Search update:");
                 sb.Append("\n");
                 sb.Append("Error: " + e.Message);
                 sb.Append("\n");
@@ -68,45 +65,51 @@ public static IHostBuilder CreateHostBuilder(string[] args) =>
                 }).ConfigureAppConfiguration((hostingContext, config) =>
                 {
                     config.AddEnvironmentVariables("TAXONOMY_");
+                    config.AddUserSecrets<Program>();
                 }).UseWindowsService();
 
 
         private static void ConfigureServicesForHost(HostBuilderContext context, IServiceCollection services)
         {
             IConfiguration config = context.Configuration;
 
-            var elasticUpdateParams = config.GetSection(nameof(ElasticUpdateParams)).Get<ElasticUpdateParams>();
-            var stagingQueueParams = config.GetSection(nameof(UpdateStagingQueueParams)).Get<UpdateStagingQueueParams>();
-            var updateElasticConnParams = config.GetSection(nameof(UpdateElasticConnectionParameters)).Get<UpdateElasticConnectionParameters>();
+            var openSearchUpdateParams = config.GetSection(nameof(OpenSearchUpdateParams)).Get<OpenSearchUpdateParams>();
 
-            services.AddSingleton(typeof(ILogger<UpdateElasticWindowsService>), typeof(Logger<UpdateElasticWindowsService>));
-            services.AddSingleton(typeof(ILogger<UpdateElasticService>), typeof(Logger<UpdateElasticService>));
+            //var stagingQueueParams = config.GetSection(nameof(UpdateStagingQueueParams)).Get<UpdateStagingQueueParams>();
+            var stagingQueueParams = config.GetSection("AmazonSqsParams").Get<AmazonSqsStagingQueueParams>();
 
+            var updateOpenSearchConnParams = config.GetSection(nameof(UpdateOpenSearchConnectionParameters)).Get<UpdateOpenSearchConnectionParameters>();
+
+            services.AddSingleton(typeof(ILogger<UpdateOpenSearchWindowsService>), typeof(Logger<UpdateOpenSearchWindowsService>));
+            services.AddSingleton(typeof(ILogger<UpdateOpenSearchService>), typeof(Logger<UpdateOpenSearchService>));
 
             //Staging queue for updates.  Needs to be a singleton or we get multiple consumers!
             services.AddSingleton<IUpdateStagingQueueReceiver>((ctx) =>
             {
-                return new ActiveMqUpdateReceiver(stagingQueueParams);
+                return new AmazonSqsUpdateReceiver(stagingQueueParams);
             });
 
-            services.AddTransient<IElasticIAViewUpdateRepository, ElasticIAViewUpdateRepository>((ctx) =>
+            services.AddTransient<IOpenSearchIAViewUpdateRepository, OpenSearchIAViewUpdateRepository>((ctx) =>
             {
-                return new ElasticIAViewUpdateRepository(updateElasticConnParams);
+                return new OpenSearchIAViewUpdateRepository(updateOpenSearchConnParams);
             });
 
-            services.AddSingleton<IUpdateElasticService>((ctx) =>
+            services.AddSingleton<IUpdateOpenSearchService>((ctx) =>
             {
-                uint bulkUpdateBatchSize = elasticUpdateParams.BulkUpdateBatchSize;
-                uint queueFetchWaitTime = elasticUpdateParams.QueueFetchSleepTime;
-                Console.WriteLine($"Using a batch size of {bulkUpdateBatchSize} and a queue fetch interval of {queueFetchWaitTime} sceonds for Elastic bulk updates.");
+                int bulkUpdateBatchSize = openSearchUpdateParams.BulkUpdateBatchSize;
+                int queueFetchWaitTime = openSearchUpdateParams.QueueFetchSleepTime;
+                int searchDatabaseUpdateInterval = openSearchUpdateParams.SearchDatabaseUpdateInterval;
+                int maxInternalQueueSize = openSearchUpdateParams.MaxInternalQueueSize;
+
+                Console.WriteLine($"Using a batch size of {bulkUpdateBatchSize} and a queue fetch interval of {queueFetchWaitTime} sceonds for Open Search bulk updates.");
 
                 IUpdateStagingQueueReceiver interimQueue = ctx.GetRequiredService<IUpdateStagingQueueReceiver>();  
-                IElasticIAViewUpdateRepository updateRepo = ctx.GetRequiredService<IElasticIAViewUpdateRepository>();
-                ILogger<UpdateElasticService> logger = ctx.GetRequiredService<ILogger<UpdateElasticService>>();
-                return new UpdateElasticService(interimQueue, updateRepo, logger, bulkUpdateBatchSize, queueFetchWaitTime);
+                IOpenSearchIAViewUpdateRepository updateRepo = ctx.GetRequiredService<IOpenSearchIAViewUpdateRepository>();
+                ILogger<UpdateOpenSearchService> logger = ctx.GetRequiredService<ILogger<UpdateOpenSearchService>>();
+                return new UpdateOpenSearchService(interimQueue, updateRepo, logger, bulkUpdateBatchSize, queueFetchWaitTime, searchDatabaseUpdateInterval, maxInternalQueueSize);
             });
 
-            services.AddHostedService<UpdateElasticWindowsService>();
+            services.AddHostedService<UpdateOpenSearchWindowsService>();
 
             ServiceProvider provider = services.BuildServiceProvider();
         }

diff --git a/NationalArchives.Taxonomy.Batch.Update.Elastic/Properties/Resources.Designer.cs b/NationalArchives.Taxonomy.Batch.Update.Elastic/Properties/Resources.Designer.cs
diff --git a/NationalArchives.Taxonomy.Batch.Update.Elastic/Properties/Resources.resx b/NationalArchives.Taxonomy.Batch.Update.Elastic/Properties/Resources.resx
@@ -117,10 +117,10 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
-  <data name="FlushRemaingUpdatesToElasticMsg" xml:space="preserve">
-    <value>Flushing remaining categorisation updates from the queue to the Elastic Search database.</value>
+  <data name="FlushRemaingUpdatesToOpenSearchMsg" xml:space="preserve">
+    <value>Flushing remaining categorisation updates from the queue to the Open Search database.</value>
   </data>
   <data name="NoMoreCategorisationResultsOnQueueMsg" xml:space="preserve">
-    <value>The Elastic update worker reports that no more categorisation results are available on the queue.  The service will now stop.</value>
+    <value>The Open Search update worker reports that no more categorisation results are available on the queue.  The service will now stop.</value>
   </data>
 </root>