From 218d98ddc6bf6a7ea3ac74dad858fd4c40c022e8 Mon Sep 17 00:00:00 2001 From: tznind Date: Thu, 25 Aug 2022 15:34:42 +0100 Subject: [PATCH 01/31] Test creating a MySql logging db from RDMP running on yamlrepository --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca37f3faba..ce5979c62f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,6 +61,8 @@ jobs: auto-start: true - name: Build run: dotnet build --configuration Release --verbosity minimal + - name: Create MySql Logging Db from YamlRepository + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ - name: Initialise RDMP run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e - name: Populate Databases.yaml From 044d5d5bcc9507de09bcdd87f842da23918298e9 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 09:08:27 +0100 Subject: [PATCH 02/31] Added logging initial creation script --- .github/workflows/build.yml | 4 ++-- Rdmp.Core/CommandExecution/BasicActivateItems.cs | 2 +- .../Versioning/MasterDatabaseScriptExecutor.cs | 3 +++ .../ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce5979c62f..ecfd16e391 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,10 +59,10 @@ jobs: mysql-version: '8.0' root-password: 'YourStrong!Passw0rd' auto-start: true - - name: Build - run: dotnet build --configuration Release --verbosity minimal - name: Create MySql Logging Db from YamlRepository run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ + - name: Build + run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- install --createdatabasetimeout 180 "(localdb)\MSSQLLocalDB" TEST_ -e - name: Populate Databases.yaml diff --git a/Rdmp.Core/CommandExecution/BasicActivateItems.cs b/Rdmp.Core/CommandExecution/BasicActivateItems.cs index bf34c4948a..6ac6db85d1 100644 --- a/Rdmp.Core/CommandExecution/BasicActivateItems.cs +++ b/Rdmp.Core/CommandExecution/BasicActivateItems.cs @@ -779,7 +779,7 @@ public virtual ExternalDatabaseServer CreateNewPlatformDatabase(ICatalogueReposi throw new ArgumentException($"Database must be picked before calling {nameof(CreateNewPlatformDatabase)} when using {nameof(BasicActivateItems)}",nameof(db)); MasterDatabaseScriptExecutor executor = new MasterDatabaseScriptExecutor(db); - executor.CreateAndPatchDatabase(patcher,new AcceptAllCheckNotifier()); + executor.CreateAndPatchDatabase(patcher, new AcceptAllCheckNotifier() { WriteToConsole = true}); var eds = new ExternalDatabaseServer(catalogueRepository,"New " + (defaultToSet == PermissableDefaults.None ? "" : defaultToSet.ToString()) + "Server",patcher); eds.SetProperties(db); diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs index 4b32f35171..9e5aed5162 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs @@ -426,6 +426,9 @@ public Patch[] GetPatchesRun() public void CreateAndPatchDatabase(IPatcher patcher, ICheckNotifier notifier) { var initialPatch = patcher.GetInitialCreateScriptContents(Database); + notifier.OnCheckPerformed( + new CheckEventArgs($"About to run:{Environment.NewLine}{initialPatch}", CheckResult.Success)); + CreateDatabase(initialPatch, notifier); //get everything in the /up/ folder that is .sql diff --git a/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs b/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs index 4a8e71d623..533978baf4 100644 --- a/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs +++ b/Reusable/ReusableLibraryCode/Checks/AcceptAllCheckNotifier.cs @@ -13,6 +13,11 @@ namespace ReusableLibraryCode.Checks /// public class AcceptAllCheckNotifier : ICheckNotifier { + /// + /// True to write out all messages seen directly to the console + /// + public bool WriteToConsole { get; set; } + /// /// Check handler that throws on Failures but otherwise returns true /// @@ -20,6 +25,9 @@ public class AcceptAllCheckNotifier : ICheckNotifier /// public virtual bool OnCheckPerformed(CheckEventArgs args) { + if (WriteToConsole) + Console.WriteLine($"{args.Result}:{args.Message}"); + //if there is a proposed fix then accept it regardless of whether it was a Fail. if (!string.IsNullOrWhiteSpace(args.ProposedFix)) return true; From 83654e3f042947b07f14670ce188aaeb09a9b39c Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 09:17:48 +0100 Subject: [PATCH 03/31] Fixed logging name of script not sql --- .../Versioning/MasterDatabaseScriptExecutor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs index 9e5aed5162..bea803e3ad 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/MasterDatabaseScriptExecutor.cs @@ -427,7 +427,7 @@ public void CreateAndPatchDatabase(IPatcher patcher, ICheckNotifier notifier) { var initialPatch = patcher.GetInitialCreateScriptContents(Database); notifier.OnCheckPerformed( - new CheckEventArgs($"About to run:{Environment.NewLine}{initialPatch}", CheckResult.Success)); + new CheckEventArgs($"About to run:{Environment.NewLine}{initialPatch.EntireScript}", CheckResult.Success)); CreateDatabase(initialPatch, notifier); From 019984d56e8328e19b3d3eca1a930fd0360b9d4a Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 09:57:35 +0100 Subject: [PATCH 04/31] Switch to C style SQL comments for version/description headers for MySql etc --- .../Databases/DataQualityEnginePatcher.cs | 2 +- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 3 +-- Rdmp.Core/Databases/QueryCachingPatcher.cs | 2 +- .../Versioning/Patch.cs | 6 +++-- .../Versioning/Patcher.cs | 23 ++++++++++++++----- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs index 8114147057..6697dd218c 100644 --- a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs +++ b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs @@ -23,7 +23,7 @@ public DataQualityEnginePatcher() : base(2, "Databases.DataQualityEngineDatabase } public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1, 0, 0)); + var header = GetHeader(db.Server.DatabaseType,InitialScriptName, new Version(1, 0, 0)); var sql = new StringBuilder(); diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index f846e2ccd1..6766755d35 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -24,9 +24,8 @@ public LoggingDatabasePatcher():base(2,"Databases.LoggingDatabase") public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1, 0, 0)); + var header = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)); - var sql = new StringBuilder(); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSet", new[] diff --git a/Rdmp.Core/Databases/QueryCachingPatcher.cs b/Rdmp.Core/Databases/QueryCachingPatcher.cs index e5e8421e0a..9d68ca0739 100644 --- a/Rdmp.Core/Databases/QueryCachingPatcher.cs +++ b/Rdmp.Core/Databases/QueryCachingPatcher.cs @@ -25,7 +25,7 @@ public QueryCachingPatcher():base(2,"Databases.QueryCachingDatabase") public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { - var header = GetHeader(InitialScriptName, new Version(1,0,0)); + var header = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1,0,0)); var body = db.Helper.GetCreateTableSql(db, "CachedAggregateConfigurationResults", new[] { diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs index 4e3c0cc307..c3f2fa9e66 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs @@ -56,10 +56,12 @@ private void ExtractDescriptionAndVersionFromScriptContents() { var lines = EntireScript.Split(new []{'\r', '\n'},StringSplitOptions.RemoveEmptyEntries); - if(!lines[0].StartsWith(VersionKey)) + var idx = lines[0].IndexOf(VersionKey); + + if (idx == -1) throw new InvalidPatchException(locationInAssembly,"Script does not start with " + VersionKey); - string versionNumber = lines[0].Substring(VersionKey.Length).Trim(':',' ','\n','\r'); + string versionNumber = lines[0].Substring(idx + VersionKey.Length).Trim(':',' ','\n','\r','\\','*'); try { diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs index c48f447df8..f29c895635 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patcher.cs @@ -43,20 +43,31 @@ protected Patcher(int tier,string resourceSubdirectory) Tier = tier; ResourceSubdirectory = resourceSubdirectory; } - + /// /// Generates a properly formatted header for creation when you only know the SQL you want to execute /// + /// /// /// /// - protected string GetHeader(string description,Version version) + protected string GetHeader(DatabaseType dbType, string description,Version version) { - string header = Patch.VersionKey + version.ToString() + Environment.NewLine; - header += Patch.DescriptionKey + description + Environment.NewLine; + string header = CommentFor(dbType,Patch.VersionKey + version.ToString()) + Environment.NewLine; + header += CommentFor(dbType,Patch.DescriptionKey + description) + Environment.NewLine; return header; } + + private string CommentFor(DatabaseType dbType, string sql) + { + if (dbType == DatabaseType.MicrosoftSQLServer) + return sql; + + // some DBMS don't like the -- notation so we need to wrap with C style comments + return $"/*{sql}*/"; + } + public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db) { var assembly = GetDbAssembly(); @@ -76,8 +87,8 @@ public virtual Patch GetInitialCreateScriptContents(DiscoveredDatabase db) string sql = sr.ReadToEnd(); - if (!sql.StartsWith(Patch.VersionKey)) - sql = GetHeader(InitialScriptName, new Version(1, 0, 0)) + sql; + if (!sql.Contains(Patch.VersionKey)) + sql = GetHeader(db.Server.DatabaseType, InitialScriptName, new Version(1, 0, 0)) + sql; return new Patch(InitialScriptName, sql); From 993b6622df062b9871c497d4449e3835c2996629 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 10:03:52 +0100 Subject: [PATCH 05/31] Fix version detection for new /**/ style comments --- Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs index c3f2fa9e66..39f44973ca 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs @@ -61,7 +61,7 @@ private void ExtractDescriptionAndVersionFromScriptContents() if (idx == -1) throw new InvalidPatchException(locationInAssembly,"Script does not start with " + VersionKey); - string versionNumber = lines[0].Substring(idx + VersionKey.Length).Trim(':',' ','\n','\r','\\','*'); + string versionNumber = lines[0].Substring(idx + VersionKey.Length).Trim(':',' ','\n','\r','/','*'); try { From 32a40ba6e0e0ac31bc7402ddb0770a23186e5491 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 10:38:06 +0100 Subject: [PATCH 06/31] Fix Description extraction to allow for C style comments --- Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs index 39f44973ca..96fbf86de9 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/Versioning/Patch.cs @@ -74,10 +74,12 @@ private void ExtractDescriptionAndVersionFromScriptContents() if(lines.Length >=2) { - if(!lines[1].StartsWith(DescriptionKey)) - throw new InvalidPatchException(locationInAssembly,"Second line of patch scripts must start with " + DescriptionKey); + idx = lines[1].IndexOf(DescriptionKey); - string description = lines[1].Substring(DescriptionKey.Length); + if (idx == -1 ) + throw new InvalidPatchException(locationInAssembly,"Second line of patch scripts must start with " + DescriptionKey); + + string description = lines[1].Substring(idx + DescriptionKey.Length).Trim(':', ' ', '\n', '\r', '/', '*'); Description = description; } } From c360b5dfbe3e05ab815b0d144a07cd2f87f73cb5 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 10:52:37 +0100 Subject: [PATCH 07/31] Add semicolon after each SQL statement in LoggingDatabasePatcher --- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index 6766755d35..96d71306e0 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -45,7 +45,8 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("contact_email",new DatabaseTypeRequest(typeof(string),64){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("frequency",new DatabaseTypeRequest(typeof(string),32){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("method",new DatabaseTypeRequest(typeof(string),16){Unicode = true}){AllowNulls = true} - }, null, false, null)); + }, null, false, null) + ";"); + // foreign keys var datasetId = new DiscoveredColumn(db.ExpectTable("DataSet"), "dataSetID", false); @@ -76,7 +77,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataLoadTask_datasetID ,datasetId } - }, true, null)); + }, true, null) + ";"); @@ -96,7 +97,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataLoadRun_dataLoadTaskID ,dataLoadTask_ID } - }, true, null)); + }, true, null) + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "TableLoadRun", new[] { @@ -117,7 +118,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {tableLoadRun_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null) + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSource", new[] { @@ -133,7 +134,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataSource_tableLoadRunID ,tableLoadRun_ID } - }, true, null)); + }, true, null) + ";"); @@ -151,7 +152,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {fatalError_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null) + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "ProgressLog", new[] @@ -166,7 +167,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {progressLog_dataLoadRunID ,dataLoadRun_ID } - }, true, null)); + }, true, null) + ";"); @@ -183,7 +184,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {rowError_tableLoadRunID ,tableLoadRun_ID } - }, true, null)); + }, true, null) + ";"); @@ -193,44 +194,44 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),50){Unicode = true }){AllowNulls = true}, new DatabaseColumnRequest("description",new DatabaseTypeRequest(typeof(string),int.MaxValue)){AllowNulls = true}, - },null, true)); + },null, true) + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_FatalErrorStatus", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true)); + }, null, true) + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_RowErrorType", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("type",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true)); + }, null, true) + ";"); sql.AppendLine(@" -INSERT INTO z_DataLoadTaskStatus(ID, status, description) VALUES(1, 'Open', NULL) -INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(2, 'Ready', NULL) -INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(3, 'Commited', NULL) -INSERT INTO z_FatalErrorStatus(ID, status) VALUES(1, 'Outstanding') -INSERT INTO z_FatalErrorStatus (ID, status) VALUES(2, 'Resolved') -INSERT INTO z_FatalErrorStatus (ID, status) VALUES(3, 'Blocked') -INSERT INTO z_RowErrorType(ID, type) VALUES(1, 'LoadRow') -INSERT INTO z_RowErrorType (ID, type) VALUES(2, 'Duplication') -INSERT INTO z_RowErrorType (ID, type) VALUES(3, 'Validation') -INSERT INTO z_RowErrorType (ID, type) VALUES(4, 'DatabaseOperation') -INSERT INTO z_RowErrorType (ID, type) VALUES(5, 'Unknown') +INSERT INTO z_DataLoadTaskStatus(ID, status, description) VALUES(1, 'Open', NULL); +INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(2, 'Ready', NULL); +INSERT INTO z_DataLoadTaskStatus (ID, status, description) VALUES(3, 'Commited', NULL); +INSERT INTO z_FatalErrorStatus(ID, status) VALUES(1, 'Outstanding'); +INSERT INTO z_FatalErrorStatus (ID, status) VALUES(2, 'Resolved'); +INSERT INTO z_FatalErrorStatus (ID, status) VALUES(3, 'Blocked'); +INSERT INTO z_RowErrorType(ID, type) VALUES(1, 'LoadRow'); +INSERT INTO z_RowErrorType (ID, type) VALUES(2, 'Duplication'); +INSERT INTO z_RowErrorType (ID, type) VALUES(3, 'Validation'); +INSERT INTO z_RowErrorType (ID, type) VALUES(4, 'DatabaseOperation'); +INSERT INTO z_RowErrorType (ID, type) VALUES(5, 'Unknown'); --create datasets -INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'DataExtraction', 'DataExtraction', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) -INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) +INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'DataExtraction', 'DataExtraction', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); --create tasks -INSERT INTO DataLoadTask(ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', GETDATE(), 'Thomas', 1, 0, 'Internal') -INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', GETDATE(), 'Thomas', 1, 0, 'DataExtraction') +INSERT INTO DataLoadTask(ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', GETDATE(), 'Thomas', 1, 0, 'Internal'); +INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', GETDATE(), 'Thomas', 1, 0, 'DataExtraction'); "); From 804b129c63bc215096506e3545051acc6f886792 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 11:02:40 +0100 Subject: [PATCH 08/31] Reduced dataSetID column length to 150 to avoid 'max key length is 767 bytes' --- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index 96d71306e0..3cb73b83a6 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -30,7 +30,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSet", new[] { - new DatabaseColumnRequest("dataSetID",new DatabaseTypeRequest(typeof(string),450){Unicode = true}){IsPrimaryKey = true}, + new DatabaseColumnRequest("dataSetID",new DatabaseTypeRequest(typeof(string),150){Unicode = true}){IsPrimaryKey = true}, new DatabaseColumnRequest("name",new DatabaseTypeRequest(typeof(string),2000){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("description",new DatabaseTypeRequest(typeof(string),int.MaxValue){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("time_period",new DatabaseTypeRequest(typeof(string),64){Unicode = true}){AllowNulls = true}, @@ -45,7 +45,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("contact_email",new DatabaseTypeRequest(typeof(string),64){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("frequency",new DatabaseTypeRequest(typeof(string),32){Unicode = true}){AllowNulls = true}, new DatabaseColumnRequest("method",new DatabaseTypeRequest(typeof(string),16){Unicode = true}){AllowNulls = true} - }, null, false, null) + ";"); + }, null, false, null).TrimEnd() + ";"); // foreign keys @@ -77,7 +77,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataLoadTask_datasetID ,datasetId } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); @@ -97,7 +97,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataLoadRun_dataLoadTaskID ,dataLoadTask_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "TableLoadRun", new[] { @@ -118,7 +118,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {tableLoadRun_dataLoadRunID ,dataLoadRun_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DataSource", new[] { @@ -134,7 +134,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {dataSource_tableLoadRunID ,tableLoadRun_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); @@ -152,7 +152,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {fatalError_dataLoadRunID ,dataLoadRun_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "ProgressLog", new[] @@ -167,7 +167,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {progressLog_dataLoadRunID ,dataLoadRun_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); @@ -184,7 +184,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {rowError_tableLoadRunID ,tableLoadRun_ID } - }, true, null) + ";"); + }, true, null).TrimEnd() + ";"); @@ -194,21 +194,21 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),50){Unicode = true }){AllowNulls = true}, new DatabaseColumnRequest("description",new DatabaseTypeRequest(typeof(string),int.MaxValue)){AllowNulls = true}, - },null, true) + ";"); + },null, true).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_FatalErrorStatus", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("status",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true) + ";"); + }, null, true).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "z_RowErrorType", new[] { new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false, IsPrimaryKey = true}, new DatabaseColumnRequest("type",new DatabaseTypeRequest(typeof(string),20){Unicode = true }), - }, null, true) + ";"); + }, null, true).TrimEnd() + ";"); sql.AppendLine(@" From 199a9d221f83bd8b5e303edfc06a9b76a0df884d Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 11:10:54 +0100 Subject: [PATCH 09/31] Switch INSERT statements to use C style comments --- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index 3cb73b83a6..9926fb2bb1 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -225,11 +225,11 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) INSERT INTO z_RowErrorType (ID, type) VALUES(4, 'DatabaseOperation'); INSERT INTO z_RowErrorType (ID, type) VALUES(5, 'Unknown'); ---create datasets +/*create datasets*/ INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'DataExtraction', 'DataExtraction', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); ---create tasks +/*create tasks*/ INSERT INTO DataLoadTask(ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', GETDATE(), 'Thomas', 1, 0, 'Internal'); INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', GETDATE(), 'Thomas', 1, 0, 'DataExtraction'); "); From 7d8d80dc33c25232230b9f075cbf8eec6131b5c5 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 11:20:54 +0100 Subject: [PATCH 10/31] Remove calls to `GETDATE()` in LoggingDatabasePatcher for MySql support --- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index 9926fb2bb1..ff3f9ee40c 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -230,8 +230,8 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) INSERT INTO DataSet (dataSetID, name, description, time_period, SLA_required, supplier_name, supplier_tel_no, supplier_email, contact_name, contact_position, currentContactInstitutions, contact_tel_no, contact_email, frequency, method) VALUES(N'Internal', 'Internal', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); /*create tasks*/ -INSERT INTO DataLoadTask(ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', GETDATE(), 'Thomas', 1, 0, 'Internal'); -INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', GETDATE(), 'Thomas', 1, 0, 'DataExtraction'); +INSERT INTO DataLoadTask(ID, description, name, userAccount, statusID, isTest, dataSetID) VALUES(1, 'Internal', 'Internal', 'Thomas', 1, 0, 'Internal'); +INSERT INTO DataLoadTask (ID, description, name, userAccount, statusID, isTest, dataSetID) VALUES(2, 'DataExtraction', 'DataExtraction', 'Thomas', 1, 0, 'DataExtraction'); "); From 99644a3be2f0c408d21bfb3a4d1fa84e323c9d2a Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 11:29:48 +0100 Subject: [PATCH 11/31] Fix foreign key dataSetID datatype to match primary key (required by Sql Server) --- Rdmp.Core/Databases/LoggingDatabasePatcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs index ff3f9ee40c..7bb789416a 100644 --- a/Rdmp.Core/Databases/LoggingDatabasePatcher.cs +++ b/Rdmp.Core/Databases/LoggingDatabasePatcher.cs @@ -73,7 +73,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("userAccount",new DatabaseTypeRequest(typeof(string),500){Unicode = true}), new DatabaseColumnRequest("statusID", new DatabaseTypeRequest(typeof(int))), new DatabaseColumnRequest("isTest", new DatabaseTypeRequest(typeof(bool))), - dataLoadTask_datasetID = new DatabaseColumnRequest("dataSetID", new DatabaseTypeRequest(typeof(string), 450) { Unicode = true }), + dataLoadTask_datasetID = new DatabaseColumnRequest("dataSetID", new DatabaseTypeRequest(typeof(string), 150) { Unicode = true }), }, new Dictionary { {dataLoadTask_datasetID ,datasetId } From fb181bce485dcfe4098e6f2577adc5fb7d9c1a04 Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 11:44:28 +0100 Subject: [PATCH 12/31] Added up front DQE and Cohort cache db testing for MySql in CI --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ecfd16e391..704f420789 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,8 +59,12 @@ jobs: mysql-version: '8.0' root-password: 'YourStrong!Passw0rd' auto-start: true - - name: Create MySql Logging Db from YamlRepository + - name: Create MySql Logging Db run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver LiveLoggingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_logging2" --dir ~/rdmp/rdmp-yaml/ + - name: Create MySql DQE Db + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver DQE "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_dqe" --dir ~/rdmp/rdmp-yaml/ + - name: Create MySql Cohort Building Query Caching Db + run: dotnet run -c Release --project Tools/rdmp/rdmp.csproj -- createnewexternaldatabaseserver CohortIdentificationQueryCachingServer_ID "DatabaseType:MySQL:Server=127.0.0.1;Uid=root;Pwd=YourStrong!Passw0rd;Database=rdmp_cache" --dir ~/rdmp/rdmp-yaml/ - name: Build run: dotnet build --configuration Release --verbosity minimal - name: Initialise RDMP From c5f677f680f356a9700a0d4c3009434d5185581d Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 12:05:47 +0100 Subject: [PATCH 13/31] Fixed `ToTier2DatabaseType` Enum value `DQE` mapping to wrong patcher type --- .../Curation/Data/Defaults/PermissableDefaultsExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs b/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs index 2caaaf93f9..ff3a5b24f9 100644 --- a/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs +++ b/Rdmp.Core/Curation/Data/Defaults/PermissableDefaultsExtensions.cs @@ -27,7 +27,7 @@ public static IPatcher ToTier2DatabaseType(this PermissableDefaults permissableD case PermissableDefaults.IdentifierDumpServer_ID: return new IdentifierDumpDatabasePatcher(); case PermissableDefaults.DQE: - return new IdentifierDumpDatabasePatcher(); + return new DataQualityEnginePatcher(); case PermissableDefaults.WebServiceQueryCachingServer_ID: return new QueryCachingPatcher(); case PermissableDefaults.CohortIdentificationQueryCachingServer_ID: From 1619ffc16edb0d630b0c14873e1928ab529b947e Mon Sep 17 00:00:00 2001 From: tznind Date: Fri, 26 Aug 2022 12:09:53 +0100 Subject: [PATCH 14/31] Add semicolons to SQL to make MySql happy --- Rdmp.Core/Databases/DataQualityEnginePatcher.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs index 6697dd218c..00ea4dc4b1 100644 --- a/Rdmp.Core/Databases/DataQualityEnginePatcher.cs +++ b/Rdmp.Core/Databases/DataQualityEnginePatcher.cs @@ -32,7 +32,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) new DatabaseColumnRequest("DateOfEvaluation",new DatabaseTypeRequest(typeof(DateTime))), new DatabaseColumnRequest("CatalogueID",new DatabaseTypeRequest(typeof(int))){AllowNulls = false}, new DatabaseColumnRequest("ID",new DatabaseTypeRequest(typeof (int))){IsAutoIncrement = true, IsPrimaryKey = true} - },null,false,null)); + },null,false,null).TrimEnd() + ";"); // foreign keys var evaluationId = new DiscoveredColumn(db.ExpectTable("Evaluation"), "ID", false); @@ -57,7 +57,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {columnState_Evaluation_ID ,evaluationId } - },true , null)); + },true , null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "RowState", new[] { @@ -72,7 +72,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {rowState_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "PeriodicityState", new[] @@ -86,7 +86,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {periodicityState_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); sql.AppendLine(db.Helper.GetCreateTableSql(db, "DQEGraphAnnotation", new[] { @@ -104,7 +104,7 @@ public override Patch GetInitialCreateScriptContents(DiscoveredDatabase db) }, new Dictionary { {annotation_Evaluation_ID ,evaluationId } - }, true, null)); + }, true, null).TrimEnd() + ";"); return new Patch(InitialScriptName, header + sql); } From e6346f61cf09165c0a6499621a2626d986831e99 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 09:45:58 +0100 Subject: [PATCH 15/31] Bump FAnsiSql to 2.0.6 --- Documentation/CodeTutorials/Packages.md | 2 +- Plugins/Plugin.Test/Plugin.Test.nuspec | 2 +- Plugins/Plugin.UI/Plugin.UI.nuspec | 2 +- Plugins/Plugin/Plugin.nuspec | 2 +- Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 3bf8fefb03..280b9611c9 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -21,7 +21,7 @@ | NPOI | [GitHub](https://github.com/tonyqus/npoi) | [2.5.5](https://www.nuget.org/packages/NPOI/2.5.5) | Apache 2.0 | Enables reading/writing Microsoft Excel files | | ExcelNumberFormat | [GitHub](https://github.com/andersnm/ExcelNumberFormat) | [1.1.0](https://www.nuget.org/packages/ExcelNumberFormat/1.1.0) |[MIT](https://opensource.org/licenses/MIT) | Handles translating number formats from Excel formats into usable values | | | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [5.0.2](https://www.nuget.org/packages/NLog/5.0.2) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.5](https://www.nuget.org/packages/HIC.FansiSql/2.0.5) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | +| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.6](https://www.nuget.org/packages/HIC.FansiSql/2.0.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | | HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [1.1.0](https://www.nuget.org/packages/HIC.BadMedicine/1.1.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses | | SSH.NET | [GitHub](https://github.com/sshnet/SSH.NET) | [2020.0.2](https://www.nuget.org/packages/SSH.NET/2020.0.2) | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers | | Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.2](https://www.nuget.org/packages/Moq/4.18.2) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt) | Mock objects during unit testing | diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 450358c048..9110920a37 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -14,7 +14,7 @@ Copyright 2018-2019 - + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index 34ebd6de23..b6c0c3e474 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -14,7 +14,7 @@ Copyright 2018-2019 - + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index d9381ce919..625c1381af 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -17,7 +17,7 @@ - + diff --git a/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj b/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj index c96291d737..a417a65ef5 100644 --- a/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj +++ b/Reusable/ReusableLibraryCode/ReusableLibraryCode.csproj @@ -57,7 +57,7 @@ - + From adeae3db0c40136a1d1ab4610542fffacfadeee4 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 09:56:44 +0100 Subject: [PATCH 16/31] Fixed wrong version of FAnsiSql in nuspec/packages.md --- Documentation/CodeTutorials/Packages.md | 2 +- Plugins/Plugin.Test/Plugin.Test.nuspec | 2 +- Plugins/Plugin.UI/Plugin.UI.nuspec | 2 +- Plugins/Plugin/Plugin.nuspec | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index 280b9611c9..bb2d8d39f6 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -21,7 +21,7 @@ | NPOI | [GitHub](https://github.com/tonyqus/npoi) | [2.5.5](https://www.nuget.org/packages/NPOI/2.5.5) | Apache 2.0 | Enables reading/writing Microsoft Excel files | | ExcelNumberFormat | [GitHub](https://github.com/andersnm/ExcelNumberFormat) | [1.1.0](https://www.nuget.org/packages/ExcelNumberFormat/1.1.0) |[MIT](https://opensource.org/licenses/MIT) | Handles translating number formats from Excel formats into usable values | | | [NLog](https://nlog-project.org/) | [GitHub](https://github.com/NLog/NLog) | [5.0.2](https://www.nuget.org/packages/NLog/5.0.2) | [BSD 3-Clause](https://github.com/NLog/NLog/blob/dev/LICENSE.txt) | Flexible user configurable logging | | -| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [2.0.6](https://www.nuget.org/packages/HIC.FansiSql/2.0.6) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | +| HIC.FAnsiSql |[GitHub](https://github.com/HicServices/FAnsiSql) | [3.0.0](https://www.nuget.org/packages/HIC.FansiSql/3.0.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | [DBMS] abstraction layer | | HIC.BadMedicine | [GitHub](https://github.com/HicServices/BadMedicine) | [1.1.0](https://www.nuget.org/packages/HIC.BadMedicine/1.1.0) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html) | Generate Test Datasets for tests/exericses | | SSH.NET | [GitHub](https://github.com/sshnet/SSH.NET) | [2020.0.2](https://www.nuget.org/packages/SSH.NET/2020.0.2) | [MIT](https://github.com/sshnet/SSH.NET/blob/develop/LICENSE) | Enables fetching files from SFTP servers | | Moq 4 | [GitHub](https://github.com/moq/moq4) | [4.18.2](https://www.nuget.org/packages/Moq/4.18.2) |[BSD 3](https://github.com/moq/moq4/blob/master/License.txt) | Mock objects during unit testing | diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 9110920a37..2a2111b97a 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -14,7 +14,7 @@ Copyright 2018-2019 - + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index b6c0c3e474..e09c26f56e 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -14,7 +14,7 @@ Copyright 2018-2019 - + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index 625c1381af..70fe228af5 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -17,7 +17,7 @@ - + From 45d957fb0c8d31d0f5f17a8e5252f14a262cf51a Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 10:55:19 +0100 Subject: [PATCH 17/31] Fixed various MySql specific keywords in logging code --- Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs | 5 ++++- Rdmp.Core.Tests/Logging/LogManagerTest.cs | 3 +++ Rdmp.Core/Logging/LogManager.cs | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs b/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs index 1a1eaab1ec..c6ddcc0163 100644 --- a/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs +++ b/Rdmp.Core.Tests/Logging/DataLoadTaskHelper.cs @@ -4,6 +4,7 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.Collections.Generic; using System.Linq; using FAnsi.Discovery; @@ -41,9 +42,11 @@ public void CreateDataLoadTask(string taskName) var taskCmd = _loggingServer.GetCommand( - "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "', GETDATE(), '" + datasetName + "', 1, 1, '" + datasetName + "')", + "INSERT INTO DataLoadTask VALUES (100, '" + taskName + "', '" + taskName + "',@date, '" + datasetName + "', 1, 1, '" + datasetName + "')", con); + _loggingServer.AddParameterWithValueToCommand("@date", taskCmd, DateTime.Now); + taskCmd.ExecuteNonQuery(); _sqlToCleanUp.Push("DELETE FROM DataLoadTask WHERE dataSetID = '" + datasetName + "'"); } diff --git a/Rdmp.Core.Tests/Logging/LogManagerTest.cs b/Rdmp.Core.Tests/Logging/LogManagerTest.cs index 7de633a10b..92c5526c51 100644 --- a/Rdmp.Core.Tests/Logging/LogManagerTest.cs +++ b/Rdmp.Core.Tests/Logging/LogManagerTest.cs @@ -238,6 +238,9 @@ public void LoggingDatabase_TestActuallyCreatingIt(DatabaseType type) Assert.AreEqual("bad.cs", archival.Errors.Single().Source); Assert.AreEqual("Wrote some records", archival.Progress.Single().Description); + + var fatal = archival.Errors.Single(); + lm.ResolveFatalErrors(new[] { fatal.ID }, DataLoadInfo.FatalErrorStates.Resolved, "problem resolved by building more towers"); } } } diff --git a/Rdmp.Core/Logging/LogManager.cs b/Rdmp.Core/Logging/LogManager.cs index 9ee1bba3c4..7ba539f196 100644 --- a/Rdmp.Core/Logging/LogManager.cs +++ b/Rdmp.Core/Logging/LogManager.cs @@ -246,10 +246,11 @@ public void CreateNewLoggingTask(int id, string dataSetID) var sql = "INSERT INTO DataLoadTask (ID, description, name, createTime, userAccount, statusID, isTest, dataSetID) " + "VALUES " + - "(" + id + ", @dataSetID, @dataSetID, GetDate(), @username, 1, 0, @dataSetID)"; + "(" + id + ", @dataSetID, @dataSetID, @date, @username, 1, 0, @dataSetID)"; using (var cmd = Server.GetCommand(sql, conn)) { + Server.AddParameterWithValueToCommand("@date", cmd,DateTime.Now); Server.AddParameterWithValueToCommand("@dataSetID",cmd,dataSetID); Server.AddParameterWithValueToCommand("@username",cmd,Environment.UserName); @@ -318,7 +319,7 @@ public void ResolveFatalErrors(int[] ids, DataLoadInfo.FatalErrorStates newState conn.Open(); { var sql = - "UPDATE [FatalError] SET explanation =@explanation, statusID=@statusID where ID in (" + string.Join(",", ids) + ")"; + "UPDATE FatalError SET explanation =@explanation, statusID=@statusID where ID in (" + string.Join(",", ids) + ")"; int affectedRows; From c01e92953f47c529973122897195e81d1cc1610c Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 10:55:35 +0100 Subject: [PATCH 18/31] Added proper wrapping to `CreateInsertStatement` --- Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs index 88db416c99..7e0f224ac0 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs @@ -640,7 +640,7 @@ private string CreateInsertStatement(Dictionary parameters) w throw new InvalidOperationException( "Invalid parameters for " + typeof(T).Name + " INSERT. Do not use @ when specifying parameter names, this is SQL-specific and will be added when required: " + string.Join(", ", parameters.Where(kvp => kvp.Key.StartsWith("@")))); - var columnString = string.Join(", ", parameters.Select(kvp => "[" + kvp.Key + "]")); + var columnString = string.Join(", ", parameters.Select(kvp => Wrap(kvp.Key))); var parameterString = string.Join(", ", parameters.Select(kvp => "@" + kvp.Key)); query += "(" + columnString + ") VALUES (" + parameterString + ")"; } @@ -961,5 +961,10 @@ public void EndTransaction(bool commit) { EndTransactedConnection(commit); } + + private string Wrap(string name) + { + return DiscoveredServer.GetQuerySyntaxHelper().EnsureWrapped(name); + } } } From 21d642b9c032912064234723f296411eb86b2523 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 11:05:01 +0100 Subject: [PATCH 19/31] Add user setting UseAliasInsteadOfTransformInGroupByAggregateGraphs --- .../MySqlAggregateBuilderTests.cs | 31 ++++++++++++++++--- Rdmp.Core/QueryBuilding/AggregateBuilder.cs | 3 +- .../Settings/UserSettings.cs | 14 +++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs index be33557d78..5dcda80250 100644 --- a/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs +++ b/Rdmp.Core.Tests/Curation/Integration/QueryBuildingTests/AggregateBuilderTests/MySqlAggregateBuilderTests.cs @@ -8,6 +8,7 @@ using NUnit.Framework; using Rdmp.Core.Curation.Data; using Rdmp.Core.QueryBuilding; +using ReusableLibraryCode.Settings; namespace Rdmp.Core.Tests.Curation.Integration.QueryBuildingTests.AggregateBuilderTests { @@ -44,21 +45,25 @@ Col1 desc topx.DeleteInDatabase(); } - [Test] - public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc() + [TestCase(true)] + [TestCase(false)] + public void Test_AggregateBuilder_MySql_Top31OrderByCountAsc(bool useAliasForGroupBy) { _ti.DatabaseType = DatabaseType.MySql; _ti.SaveToDatabase(); + UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = useAliasForGroupBy; + var builder = new AggregateBuilder(null, "count(*)", null); builder.AddColumn(_dimension1); var topx = new AggregateTopX(CatalogueRepository, _configuration, 31); topx.OrderByDirection = AggregateTopXOrderByDirection.Ascending; builder.AggregateTopX = topx; - - Assert.AreEqual(CollapseWhitespace(@"/**/ + if (useAliasForGroupBy) + { + Assert.AreEqual(CollapseWhitespace(@"/**/ SELECT Col1, count(*) AS MyCount @@ -69,9 +74,27 @@ group by order by MyCount asc LIMIT 31"), CollapseWhitespace(builder.SQL)); + } + else + { + Assert.AreEqual(CollapseWhitespace(@"/**/ +SELECT +Col1, +count(*) AS MyCount +FROM +T1 +group by +Col1 +order by +count(*) asc +LIMIT 31"), CollapseWhitespace(builder.SQL)); + } + topx.DeleteInDatabase(); + + UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs = false; } } } diff --git a/Rdmp.Core/QueryBuilding/AggregateBuilder.cs b/Rdmp.Core/QueryBuilding/AggregateBuilder.cs index 7a1266fdf9..7f78175136 100644 --- a/Rdmp.Core/QueryBuilding/AggregateBuilder.cs +++ b/Rdmp.Core/QueryBuilding/AggregateBuilder.cs @@ -13,6 +13,7 @@ using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.QueryBuilding.Options; using Rdmp.Core.QueryBuilding.Parameters; +using ReusableLibraryCode.Settings; namespace Rdmp.Core.QueryBuilding { @@ -564,7 +565,7 @@ private string GetGroupOrOrderByCustomLineBasedOn(QueryTimeColumn col) private string GetGroupOrOrderByCustomLineBasedOn(string select, string alias) { - if (QuerySyntaxHelper.DatabaseType == FAnsi.DatabaseType.MySql) + if (UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs) { return !string.IsNullOrWhiteSpace(alias) ? alias : // for MySql prefer using the alias if it has one diff --git a/Reusable/ReusableLibraryCode/Settings/UserSettings.cs b/Reusable/ReusableLibraryCode/Settings/UserSettings.cs index 844e0529b1..aa3a9df2c2 100644 --- a/Reusable/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Reusable/ReusableLibraryCode/Settings/UserSettings.cs @@ -397,6 +397,20 @@ public static bool ShowProjectSpecificColumns set { AppSettings.AddOrUpdateValue("ShowProjectSpecificColumns", value); } } + /// + /// When generating an aggregate graph, use the column alias instead of the select sql. For example + /// when you have the select column 'SELECT YEAR(dt) as myYear' then the GROUP BY will default to + /// 'GROUP BY YEAR(dt)'. Setting this property to true will instead use 'GROUP BY myYear'. Typically + /// this only works in MySql but it is not universally supported by all MySql versions and server settings + /// + /// Defaults to false. + /// + public static bool UseAliasInsteadOfTransformInGroupByAggregateGraphs + { + get { return AppSettings.GetValueOrDefault("ShowProjectSpecificColumns", false); } + set { AppSettings.AddOrUpdateValue("ShowProjectSpecificColumns", value); } + } + #endregion From 3a36e9cdb6ae1dd2fbd01e171b7c3cfa4ef0945f Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 11:59:53 +0100 Subject: [PATCH 20/31] Fix sql server specific objects in logging API --- Rdmp.Core/Logging/DataLoadInfo.cs | 89 ++++++++-------------- Rdmp.Core/Logging/TableLoadInfo.cs | 105 +++++++++----------------- Rdmp.Core/Repositories/RowVerCache.cs | 2 +- 3 files changed, 67 insertions(+), 129 deletions(-) diff --git a/Rdmp.Core/Logging/DataLoadInfo.cs b/Rdmp.Core/Logging/DataLoadInfo.cs index b585dd1e4d..76df9951c8 100644 --- a/Rdmp.Core/Logging/DataLoadInfo.cs +++ b/Rdmp.Core/Logging/DataLoadInfo.cs @@ -6,9 +6,7 @@ using System; using System.Collections.Generic; -using System.Data; using System.Data.Common; -using Microsoft.Data.SqlClient; using FAnsi.Discovery; namespace Rdmp.Core.Logging @@ -160,15 +158,12 @@ public DataLoadInfo(string dataLoadTaskName, string packageName, string descript private void RecordNewDataLoadInDatabase(string dataLoadTaskName) { - int parentTaskID = -1; - - using (var con = (SqlConnection)_server.GetConnection()) + using (var con = _server.GetConnection()) { con.Open(); - SqlCommand cmd = new SqlCommand("SELECT ID FROM DataLoadTask WHERE name=@name", con); - cmd.Parameters.Add("@name", SqlDbType.VarChar, 255); - cmd.Parameters["@name"].Value = dataLoadTaskName; + var cmd = _server.GetCommand("SELECT ID FROM DataLoadTask WHERE name=@name", con); + _server.AddParameterWithValueToCommand("@name",cmd, dataLoadTaskName); var result = cmd.ExecuteScalar(); @@ -177,28 +172,20 @@ private void RecordNewDataLoadInDatabase(string dataLoadTaskName) throw new Exception("Could not find data load task named:" + dataLoadTaskName); //ID can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it - parentTaskID = int.Parse(result.ToString()); + var parentTaskID = int.Parse(result.ToString()); + + cmd = _server.GetCommand( + @"INSERT INTO DataLoadRun (description,startTime,dataLoadTaskID,isTest,packageName,userAccount,suggestedRollbackCommand) VALUES (@description,@startTime,@dataLoadTaskID,@isTest,@packageName,@userAccount,@suggestedRollbackCommand); +SELECT @@IDENTITY;", con); + _server.AddParameterWithValueToCommand("@description", cmd, _description); + _server.AddParameterWithValueToCommand("@startTime", cmd, _startTime); + _server.AddParameterWithValueToCommand("@dataLoadTaskID", cmd, parentTaskID); + _server.AddParameterWithValueToCommand("@isTest",cmd, _isTest); + _server.AddParameterWithValueToCommand("@packageName", cmd, _packageName); + _server.AddParameterWithValueToCommand("@userAccount", cmd, _userAccount); + _server.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand ?? string.Empty); - cmd = new SqlCommand( - @"INSERT INTO DataLoadRun (description,startTime,dataLoadTaskID,isTest,packageName,userAccount,suggestedRollbackCommand) VALUES (@description,@startTime,@dataLoadTaskID,@isTest,@packageName,@userAccount,@suggestedRollbackCommand); -SELECT SCOPE_IDENTITY();", con); - - cmd.Parameters.Add("@description", SqlDbType.VarChar, -1); - cmd.Parameters.Add("@startTime", SqlDbType.DateTime); - cmd.Parameters.Add("@dataLoadTaskID", SqlDbType.Int); - cmd.Parameters.Add("@isTest", SqlDbType.Bit); - cmd.Parameters.Add("@packageName", SqlDbType.VarChar, 100); - cmd.Parameters.Add("@userAccount", SqlDbType.VarChar, 50); - cmd.Parameters.Add("@suggestedRollbackCommand", SqlDbType.VarChar, -1); - - cmd.Parameters["@description"].Value = _description; - cmd.Parameters["@startTime"].Value = _startTime; - cmd.Parameters["@dataLoadTaskID"].Value = parentTaskID; - cmd.Parameters["@isTest"].Value = _isTest; - cmd.Parameters["@packageName"].Value = _packageName; - cmd.Parameters["@userAccount"].Value = _userAccount; - cmd.Parameters["@suggestedRollbackCommand"].Value = _suggestedRollbackCommand ?? string.Empty; //ID can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it _id = int.Parse(cmd.ExecuteScalar().ToString()); @@ -312,25 +299,19 @@ public void LogFatalError(string errorSource, string errorDescription) //look up the fatal error ID (get hte name of the Enum so that we can refactor if nessesary without breaking the code looking for a constant string) string initialErrorStatus = Enum.GetName(typeof(FatalErrorStates), FatalErrorStates.Outstanding); - SqlCommand cmdLookupStatusID = new SqlCommand("SELECT ID from z_FatalErrorStatus WHERE status=@status", (SqlConnection)con); - cmdLookupStatusID.Parameters.Add("@status", SqlDbType.NChar, 20); - cmdLookupStatusID.Parameters["@status"].Value = initialErrorStatus; + + var cmdLookupStatusID = _server.GetCommand("SELECT ID from z_FatalErrorStatus WHERE status=@status", con); + _server.AddParameterWithValueToCommand("@status",cmdLookupStatusID, initialErrorStatus); int statusID = int.Parse(cmdLookupStatusID.ExecuteScalar().ToString()); - SqlCommand cmdRecordFatalError = new SqlCommand( - @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", (SqlConnection)con); - cmdRecordFatalError.Parameters.Add("@time", SqlDbType.DateTime); - cmdRecordFatalError.Parameters.Add("@source", SqlDbType.VarChar, 50); - cmdRecordFatalError.Parameters.Add("@description", SqlDbType.VarChar, -1); - cmdRecordFatalError.Parameters.Add("@statusID", SqlDbType.Int); - cmdRecordFatalError.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - - cmdRecordFatalError.Parameters["@time"].Value = DateTime.Now; - cmdRecordFatalError.Parameters["@source"].Value = errorSource; - cmdRecordFatalError.Parameters["@description"].Value = errorDescription; - cmdRecordFatalError.Parameters["@statusID"].Value = statusID; - cmdRecordFatalError.Parameters["@dataLoadRunID"].Value = ID; + var cmdRecordFatalError = _server.GetCommand( + @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", con); + _server.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); + _server.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); + _server.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); + _server.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); + _server.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); cmdRecordFatalError.ExecuteNonQuery(); @@ -352,24 +333,18 @@ public enum ProgressEventType public void LogProgress(ProgressEventType pevent, string Source, string Description) { - using (var con = (SqlConnection)DatabaseSettings.GetConnection()) - using (var cmdRecordProgress = new SqlCommand("INSERT INTO ProgressLog " + + using (var con = DatabaseSettings.GetConnection()) + using (var cmdRecordProgress = _server.GetCommand("INSERT INTO ProgressLog " + "(dataLoadRunID,eventType,source,description,time) " + "VALUES (@dataLoadRunID,@eventType,@source,@description,@time);", con)) { con.Open(); - cmdRecordProgress.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - cmdRecordProgress.Parameters.Add("@eventType", SqlDbType.VarChar, 50); - cmdRecordProgress.Parameters.Add("@source", SqlDbType.VarChar, 100); - cmdRecordProgress.Parameters.Add("@description", SqlDbType.VarChar, 8000); - cmdRecordProgress.Parameters.Add("@time", SqlDbType.DateTime); - - cmdRecordProgress.Parameters["@dataLoadRunID"].Value = ID; - cmdRecordProgress.Parameters["@eventType"].Value = pevent.ToString(); - cmdRecordProgress.Parameters["@source"].Value = Source; - cmdRecordProgress.Parameters["@description"].Value = Description; - cmdRecordProgress.Parameters["@time"].Value = DateTime.Now; + _server.AddParameterWithValueToCommand("@dataLoadRunID",cmdRecordProgress, ID); + _server.AddParameterWithValueToCommand("@eventType", cmdRecordProgress, pevent.ToString()); + _server.AddParameterWithValueToCommand("@source", cmdRecordProgress, Source); + _server.AddParameterWithValueToCommand("@description", cmdRecordProgress, Description); + _server.AddParameterWithValueToCommand("@time", cmdRecordProgress, DateTime.Now); cmdRecordProgress.ExecuteNonQuery(); } diff --git a/Rdmp.Core/Logging/TableLoadInfo.cs b/Rdmp.Core/Logging/TableLoadInfo.cs index 4490d7ba3f..82d00e3c25 100644 --- a/Rdmp.Core/Logging/TableLoadInfo.cs +++ b/Rdmp.Core/Logging/TableLoadInfo.cs @@ -6,7 +6,7 @@ using System; using System.Data; -using Microsoft.Data.SqlClient; +using FAnsi.Connections; using FAnsi.Discovery; namespace Rdmp.Core.Logging @@ -78,23 +78,18 @@ public TableLoadInfo(DataLoadInfo parent,string suggestedRollbackCommand,string private void RecordNewTableLoadInDatabase(DataLoadInfo parent,string destinationTable, DataSource[] sources, int expectedInserts) { - using (var con = (SqlConnection)_databaseSettings.GetConnection()) - using (var cmd = new SqlCommand("INSERT INTO TableLoadRun (startTime,dataLoadRunID,targetTable,expectedInserts,suggestedRollbackCommand) " + + using (var con = _databaseSettings.GetConnection()) + using (var cmd = _databaseSettings.GetCommand("INSERT INTO TableLoadRun (startTime,dataLoadRunID,targetTable,expectedInserts,suggestedRollbackCommand) " + "VALUES (@startTime,@dataLoadRunID,@targetTable,@expectedInserts,@suggestedRollbackCommand); " + - "SELECT SCOPE_IDENTITY();", con)) + "SELECT @@IDENTITY;", con)) { con.Open(); - cmd.Parameters.Add("@startTime", SqlDbType.DateTime); - cmd.Parameters.Add("@dataLoadRunID", SqlDbType.Int); - cmd.Parameters.Add("@targetTable", SqlDbType.VarChar, 200); - cmd.Parameters.Add("@expectedInserts", SqlDbType.BigInt); - cmd.Parameters.Add("@suggestedRollbackCommand", SqlDbType.VarChar, -1); - - cmd.Parameters["@startTime"].Value = DateTime.Now; - cmd.Parameters["@dataLoadRunID"].Value = parent.ID; - cmd.Parameters["@targetTable"].Value = destinationTable; - cmd.Parameters["@expectedInserts"].Value = expectedInserts; - cmd.Parameters["@suggestedRollbackCommand"].Value = _suggestedRollbackCommand; + + _databaseSettings.AddParameterWithValueToCommand("@startTime",cmd, DateTime.Now); + _databaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmd, parent.ID); + _databaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable); + _databaseSettings.AddParameterWithValueToCommand("@expectedInserts", cmd, expectedInserts); + _databaseSettings.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand); //get the ID, can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it _id = int.Parse(cmd.ExecuteScalar().ToString()); @@ -105,27 +100,14 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent,string destination //for each of the sources, create them in the DataSource table foreach (DataSource s in DataSources) { - using (var cmdInsertDs = new SqlCommand("INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + - "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT SCOPE_IDENTITY();", con)) + using (var cmdInsertDs = _databaseSettings.GetCommand("INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + + "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con)) { - cmdInsertDs.Parameters.Add("@source", SqlDbType.VarChar, -1); - cmdInsertDs.Parameters.Add("@tableLoadRunID", SqlDbType.Int); - cmdInsertDs.Parameters.Add("@originDate", SqlDbType.Date); - cmdInsertDs.Parameters.Add("@MD5", SqlDbType.Binary, 128); - - - cmdInsertDs.Parameters["@source"].Value = s.Source; - cmdInsertDs.Parameters["@tableLoadRunID"].Value = _id; - - if (s.UnknownOriginDate) - cmdInsertDs.Parameters["@originDate"].Value = DBNull.Value; - else - cmdInsertDs.Parameters["@originDate"].Value = s.OriginDate; - if (s.MD5 != null) - cmdInsertDs.Parameters["@MD5"].Value = s.MD5; - else - cmdInsertDs.Parameters["@MD5"].Value = DBNull.Value; + _databaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); + _databaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); + _databaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, s.UnknownOriginDate ? DBNull.Value : s.OriginDate); + _databaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, s.MD5 != null ? s.MD5:DBNull.Value); s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); } @@ -220,36 +202,20 @@ public string Notes public void CloseAndArchive() { - using (var con = (SqlConnection)_databaseSettings.GetConnection()) + using (var con = _databaseSettings.BeginNewTransactedConnection()) { - con.Open(); - using (SqlTransaction transaction = con.BeginTransaction()) - using (var cmdCloseRecord = new SqlCommand("UPDATE TableLoadRun SET endTime=@endTime,inserts=@inserts,updates=@updates,deletes=@deletes,errorRows=@errorRows,duplicates=@duplicates, notes=@notes WHERE ID=@ID", con, transaction)) + using (var cmdCloseRecord = _databaseSettings.GetCommand("UPDATE TableLoadRun SET endTime=@endTime,inserts=@inserts,updates=@updates,deletes=@deletes,errorRows=@errorRows,duplicates=@duplicates, notes=@notes WHERE ID=@ID", con.Connection, con.ManagedTransaction)) { try { - cmdCloseRecord.Parameters.Add("@endTime", SqlDbType.DateTime); - cmdCloseRecord.Parameters.Add("@inserts", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@updates", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@deletes", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@errorRows", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@duplicates", SqlDbType.BigInt); - cmdCloseRecord.Parameters.Add("@notes", SqlDbType.VarChar); - cmdCloseRecord.Parameters.Add("@ID", SqlDbType.Int); - - cmdCloseRecord.Parameters["@endTime"].Value = DateTime.Now; - cmdCloseRecord.Parameters["@inserts"].Value = this.Inserts; - cmdCloseRecord.Parameters["@updates"].Value = this.Updates; - cmdCloseRecord.Parameters["@deletes"].Value = this.Deletes; - cmdCloseRecord.Parameters["@errorRows"].Value = this.ErrorRows; - cmdCloseRecord.Parameters["@duplicates"].Value = this.DiscardedDuplicates; - - if (string.IsNullOrWhiteSpace(this.Notes)) - cmdCloseRecord.Parameters["@notes"].Value = DBNull.Value; - else - cmdCloseRecord.Parameters["@notes"].Value = this.Notes; - - cmdCloseRecord.Parameters["@ID"].Value = this.ID; + _databaseSettings.AddParameterWithValueToCommand("@endTime",cmdCloseRecord, DateTime.Now); + _databaseSettings.AddParameterWithValueToCommand("@inserts", cmdCloseRecord, this.Inserts); + _databaseSettings.AddParameterWithValueToCommand("@updates", cmdCloseRecord, this.Updates); + _databaseSettings.AddParameterWithValueToCommand("@deletes", cmdCloseRecord, this.Deletes); + _databaseSettings.AddParameterWithValueToCommand("@errorRows", cmdCloseRecord, this.ErrorRows); + _databaseSettings.AddParameterWithValueToCommand("@duplicates", cmdCloseRecord, this.DiscardedDuplicates); + _databaseSettings.AddParameterWithValueToCommand("@notes", cmdCloseRecord, string.IsNullOrWhiteSpace(this.Notes) ? DBNull.Value : this.Notes); + _databaseSettings.AddParameterWithValueToCommand("@ID", cmdCloseRecord, this.ID); int affectedRows = cmdCloseRecord.ExecuteNonQuery(); @@ -257,9 +223,9 @@ public void CloseAndArchive() throw new Exception("Error closing TableLoadInfo in database, the UPDATE command affected " + affectedRows + " when we expected 1 (will attempt to rollback transaction)"); foreach (DataSource s in DataSources) - MarkDataSourceAsArchived(s, con, transaction); + MarkDataSourceAsArchived(s, con); - transaction.Commit(); + con.ManagedTransaction.CommitAndCloseConnection(); _endTime = DateTime.Now; _isClosed = true; @@ -267,27 +233,24 @@ public void CloseAndArchive() catch (Exception) { //if something goes wrong with the update, roll it back - transaction.Rollback(); + con.ManagedTransaction.AbandonAndCloseConnection(); throw; } } } } - private void MarkDataSourceAsArchived(DataSource ds, SqlConnection con, SqlTransaction transaction) + private void MarkDataSourceAsArchived(DataSource ds, IManagedConnection con) { if (string.IsNullOrEmpty(ds.Archive)) return; - using (var cmdSetArchived = new SqlCommand("UPDATE DataSource SET archive=@archive, source = @source WHERE ID=@ID", con, transaction)) + using (var cmdSetArchived = _databaseSettings.GetCommand("UPDATE DataSource SET archive=@archive, source = @source WHERE ID=@ID", con.Connection, con.ManagedTransaction)) { - cmdSetArchived.Parameters.Add("@archive", SqlDbType.VarChar, -1); - cmdSetArchived.Parameters.Add("@source", SqlDbType.VarChar, -1); - cmdSetArchived.Parameters.Add("@ID", SqlDbType.Int); - cmdSetArchived.Parameters["@archive"].Value = ds.Archive; - cmdSetArchived.Parameters["@source"].Value = ds.Source; - cmdSetArchived.Parameters["@ID"].Value = ds.ID; + _databaseSettings.AddParameterWithValueToCommand("@archive", cmdSetArchived,ds.Archive); + _databaseSettings.AddParameterWithValueToCommand("@source", cmdSetArchived, ds.Source); + _databaseSettings.AddParameterWithValueToCommand("@ID", cmdSetArchived, ds.ID); cmdSetArchived.ExecuteNonQuery(); } diff --git a/Rdmp.Core/Repositories/RowVerCache.cs b/Rdmp.Core/Repositories/RowVerCache.cs index 7a578b725e..b8383d2ee4 100644 --- a/Rdmp.Core/Repositories/RowVerCache.cs +++ b/Rdmp.Core/Repositories/RowVerCache.cs @@ -6,11 +6,11 @@ using System; using System.Collections.Generic; -using Microsoft.Data.SqlClient; using System.Linq; using System.Text; using System.Threading; using MapsDirectlyToDatabaseTable; +using Microsoft.Data.SqlClient; namespace Rdmp.Core.Repositories { From 283efa255a20fe8ddc0ef563e02ebc15f2776303 Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 16:28:04 +0100 Subject: [PATCH 21/31] Fixed GetArchivalDataLoadInfos to have MySql comaptible ANSI syntax --- Rdmp.Core/Logging/LogManager.cs | 33 +++++++++++++++---- .../PastEvents/ArchivalDataLoadInfo.cs | 5 ++- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/Logging/LogManager.cs b/Rdmp.Core/Logging/LogManager.cs index 7ba539f196..13fa6859c5 100644 --- a/Rdmp.Core/Logging/LogManager.cs +++ b/Rdmp.Core/Logging/LogManager.cs @@ -9,9 +9,11 @@ using System.Data; using System.Data.Common; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using FAnsi.Discovery; +using FAnsi.Discovery.QuerySyntax; using Rdmp.Core.Logging.PastEvents; using ReusableLibraryCode; using ReusableLibraryCode.DataAccess; @@ -153,13 +155,9 @@ public IEnumerable GetArchivalDataLoadInfos(string dataTas var dataTaskId = GetDataTaskId(dataTask,Server, con); string where = ""; - string top = ""; using (var cmd = Server.GetCommand("", con)) { - if (topX != null) - top = "TOP " + topX.Value; - if (specificDataLoadRunIDOnly != null) where = "WHERE ID=" + specificDataLoadRunIDOnly.Value; else @@ -171,9 +169,32 @@ public IEnumerable GetArchivalDataLoadInfos(string dataTas cmd.Parameters.Add(p); } - string sql = "SELECT " + top + " *, (select top 1 1 from FatalError where dataLoadRunID = DataLoadRun.ID) hasErrors FROM " + run.GetFullyQualifiedName() +" " + where + " ORDER BY ID desc"; + TopXResponse top = null; + + if (topX.HasValue) + top = Server.GetQuerySyntaxHelper().HowDoWeAchieveTopX(topX.Value); + + StringBuilder sb = new StringBuilder(); + + + sb.Append("SELECT "); + + if(top?.Location == QueryComponent.SELECT) + { + sb.AppendLine(top.SQL); + } + + sb.Append(" *"); + + + sb.AppendLine($" FROM {run.GetFullyQualifiedName()} {where} ORDER BY ID desc"); + + if(top?.Location == QueryComponent.Postfix) + { + sb.AppendLine(top.SQL); + } - cmd.CommandText = sql; + cmd.CommandText = sb.ToString(); DbDataReader r; if (token == null) diff --git a/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs b/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs index 7f527568f5..f484a16bfc 100644 --- a/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs +++ b/Rdmp.Core/Logging/PastEvents/ArchivalDataLoadInfo.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Data.Common; +using System.Linq; using FAnsi.Discovery; using ReusableLibraryCode.Settings; @@ -28,7 +29,7 @@ public class ArchivalDataLoadInfo : IArchivalLoggingRecordOfPastEvent, IComparab public DateTime StartTime { get; internal set; } public DateTime? EndTime { get; internal set; } - public bool HasErrors { get; private set; } + public bool HasErrors => _knownErrors.Value.Any(); public string ToShortString() { @@ -94,8 +95,6 @@ internal ArchivalDataLoadInfo(DbDataReader r,DiscoveredDatabase loggingDatabase) Description = r["description"] as string; - HasErrors = r["hasErrors"] != DBNull.Value; - _knownTableInfos = new Lazy>(GetTableInfos); _knownErrors = new Lazy>(GetErrors); _knownProgress = new Lazy>(GetProgress); From dd130ff3b09111ce0a722d90d7cdb798d99c095f Mon Sep 17 00:00:00 2001 From: tznind Date: Mon, 29 Aug 2022 17:13:15 +0100 Subject: [PATCH 22/31] Remove Sql Server specific stuff in DQE API --- Rdmp.Core/DataQualityEngine/Data/ColumnState.cs | 2 +- .../DataQualityEngine/Data/PeriodicityState.cs | 15 ++++++++++----- Rdmp.Core/DataQualityEngine/Data/RowState.cs | 2 +- .../TableRepository.cs | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs index 1449d90973..35f8df8c7d 100644 --- a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs @@ -133,7 +133,7 @@ public void Commit(Evaluation evaluation,string pivotCategory, DbConnection con, throw new NotSupportedException("ColumnState was already committed"); var sql = string.Format( - "INSERT INTO [dbo].[ColumnState]([TargetProperty],[DataLoadRunID],[Evaluation_ID],[CountCorrect],[CountDBNull],[ItemValidatorXML],[CountMissing],[CountWrong],[CountInvalidatesRow],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", + "INSERT INTO dbo.ColumnState(TargetProperty,DataLoadRunID,Evaluation_ID,CountCorrect,CountDBNull,ItemValidatorXML,CountMissing,CountWrong,CountInvalidatesRow,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", "@TargetProperty", DataLoadRunID ,evaluation.ID diff --git a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs index 11b50654c2..54abf777a7 100644 --- a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Data; +using MapsDirectlyToDatabaseTable; using Rdmp.Core.Validation.Constraints; using ReusableLibraryCode; @@ -56,17 +57,19 @@ public static Dictionary GetPeriodicityCount var calc = new DatasetTimespanCalculator(); var result = calc.GetMachineReadableTimespanIfKnownOf(evaluation, discardOutliers); + var t = evaluation.DQERepository; + using (var con = evaluation.DQERepository.GetConnection()) { var sql = - @"SELECT - [Year] - ,[Month] + @$"SELECT + {t.Wrap("Year")} + ,{t.Wrap("Month")} ,RowEvaluation ,CountOfRecords FROM [PeriodicityState] where - Evaluation_ID = " + evaluation.ID + " and PivotCategory = 'ALL' ORDER BY [Year],[Month]"; + Evaluation_ID = ${evaluation.ID} and PivotCategory = 'ALL' ORDER BY {t.Wrap("Year")},{t.Wrap("Month")}"; using(var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction)) { @@ -179,9 +182,11 @@ public void Commit(Evaluation evaluation, string pivotCategory) if (IsCommitted) throw new NotSupportedException("PeriodicityState was already committed"); + var t = evaluation.DQERepository; + string sql = string.Format( - "INSERT INTO [dbo].[PeriodicityState]([Evaluation_ID],[Year],[Month],[CountOfRecords],[RowEvaluation],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5})" + $"INSERT INTO PeriodicityState(Evaluation_ID,{t.Wrap("Year")},{t.Wrap("Month")},CountOfRecords,RowEvaluation,PivotCategory)VALUES({0},{1},{2},{3},{4},{5})" ,evaluation.ID ,Year ,Month diff --git a/Rdmp.Core/DataQualityEngine/Data/RowState.cs b/Rdmp.Core/DataQualityEngine/Data/RowState.cs index 9225bee892..64d61ea59e 100644 --- a/Rdmp.Core/DataQualityEngine/Data/RowState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/RowState.cs @@ -43,7 +43,7 @@ public RowState(Evaluation evaluation, int dataLoadRunID, int correct, int missi { var sql = string.Format( - "INSERT INTO [dbo].[RowState]([Evaluation_ID],[Correct],[Missing],[Wrong],[Invalid],[DataLoadRunID],[ValidatorXML],[PivotCategory])VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})", + "INSERT INTO RowState(Evaluation_ID,Correct,Missing,Wrong,Invalid,DataLoadRunID,ValidatorXML,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},@validatorXML,{6})", evaluation.ID, correct, missing, diff --git a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs index 7e0f224ac0..65bdff3b7d 100644 --- a/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs +++ b/Reusable/MapsDirectlyToDatabaseTable/TableRepository.cs @@ -962,7 +962,7 @@ public void EndTransaction(bool commit) EndTransactedConnection(commit); } - private string Wrap(string name) + public string Wrap(string name) { return DiscoveredServer.GetQuerySyntaxHelper().EnsureWrapped(name); } From 677cb604c296d7ce7de52027c0e765f872622171 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 08:08:32 +0100 Subject: [PATCH 23/31] Fixed bad sql and dodgy string interpolation/format mixing --- Rdmp.Core/DataQualityEngine/Data/ColumnState.cs | 2 +- Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs index 35f8df8c7d..810abe41da 100644 --- a/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/ColumnState.cs @@ -133,7 +133,7 @@ public void Commit(Evaluation evaluation,string pivotCategory, DbConnection con, throw new NotSupportedException("ColumnState was already committed"); var sql = string.Format( - "INSERT INTO dbo.ColumnState(TargetProperty,DataLoadRunID,Evaluation_ID,CountCorrect,CountDBNull,ItemValidatorXML,CountMissing,CountWrong,CountInvalidatesRow,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", + "INSERT INTO ColumnState(TargetProperty,DataLoadRunID,Evaluation_ID,CountCorrect,CountDBNull,ItemValidatorXML,CountMissing,CountWrong,CountInvalidatesRow,PivotCategory)VALUES({0},{1},{2},{3},{4},{5},{6},{7},{8},{9})", "@TargetProperty", DataLoadRunID ,evaluation.ID diff --git a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs index 54abf777a7..048ca45d47 100644 --- a/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs +++ b/Rdmp.Core/DataQualityEngine/Data/PeriodicityState.cs @@ -185,14 +185,7 @@ public void Commit(Evaluation evaluation, string pivotCategory) var t = evaluation.DQERepository; string sql = - string.Format( - $"INSERT INTO PeriodicityState(Evaluation_ID,{t.Wrap("Year")},{t.Wrap("Month")},CountOfRecords,RowEvaluation,PivotCategory)VALUES({0},{1},{2},{3},{4},{5})" - ,evaluation.ID - ,Year - ,Month - ,CountOfRecords - , "@RowEvaluation", - "@PivotCategory"); + $"INSERT INTO PeriodicityState(Evaluation_ID,{t.Wrap("Year")},{t.Wrap("Month")},CountOfRecords,RowEvaluation,PivotCategory)VALUES({evaluation.ID},{Year},{Month},{CountOfRecords},@RowEvaluation,@PivotCategory)"; using (var cmd = DatabaseCommandHelper.GetCommand(sql, con.Connection, con.Transaction)) { From 4931fd933675634ff6a34f0947884b8647c2a753 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 09:19:16 +0100 Subject: [PATCH 24/31] Added GetDate to Ignored word list for doc rot test --- .../ClassFileEvaluation/DocumentationCrossExaminationTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs index 6f3289db68..bfea41917d 100644 --- a/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs +++ b/Rdmp.UI.Tests/DesignPatternTests/ClassFileEvaluation/DocumentationCrossExaminationTest.cs @@ -261,7 +261,8 @@ class DocumentationCrossExaminationTest "NuGet", "MyPluginClass", "SubContainer", - "DescribeCommand" // this class has now been removed from RDMP codebase, don't complain if you see it in docs (e.g. CHANGELOG.md). + "DescribeCommand", // this class has now been removed from RDMP codebase, don't complain if you see it in docs (e.g. CHANGELOG.md). + "GetDate" }; #endregion public DocumentationCrossExaminationTest(DirectoryInfo slndir) From 4920d7dedfd9f010633826c4f30126a4c065da64 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 09:48:37 +0100 Subject: [PATCH 25/31] The 'Core' folder in extraction execution user interface is no longer disabled when empty --- CHANGELOG.md | 8 +++----- Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs | 6 +----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c47a20b537..f2bc67ba75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ... -### Changed -- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307) -- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365) - ### Added - Added 'Set Description' command to [AggregateConfiguration] context menu - Template cohort builder aggregates can be dragged onto extraction datasets to import the container tree [#1307](https://github.com/HicServices/RDMP/issues/1307) @@ -20,8 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added ability to pop out tooltips/problems into modal popup [#1334](https://github.com/HicServices/RDMP/issues/1334) ### Changed - +- The 'Core' folder in extraction execution user interface is no longer disabled when empty [#1377](https://github.com/HicServices/RDMP/issues/1377) - Datasets in extraction UI are no longer expanded by default (i.e. to show Supporting Documents/Sql) [#1264](https://github.com/HicServices/RDMP/issues/1264) +- Removed restriction preventing [Lookup] requiring all foreign key columns being from the same table [#1331](https://github.com/HicServices/RDMP/issues/1307) +- If there are multiple IsPrimaryExtractionTable involved in a query then the one with the IsExtractionIdentifier column (if any) will be picked (previously QueryBuildingException was thrown) [#1365](https://github.com/HicServices/RDMP/issues/1365) ### Fixed diff --git a/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs b/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs index d9f0277ecb..59a9383740 100644 --- a/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs +++ b/Rdmp.UI/ProjectUI/ExecuteExtractionUI.cs @@ -286,11 +286,7 @@ public override void SetDatabaseObject(IActivateItems activator, ExtractionConfi //if there are no project specific datasets if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID == null)) tlvDatasets.DisableObject(_projectSpecificDatasetsFolder); //disable this option - - //if all the datasets are project specific - if (_datasets.All(sds => sds.ExtractableDataSet.Project_ID != null)) - tlvDatasets.DisableObject(_coreDatasetsFolder); - + //don't accept refresh while executing if (checkAndExecuteUI1.IsExecuting) return; From 720934d5aab789fa4cc57a4787cbbd8bfa6373b4 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 10:49:45 +0100 Subject: [PATCH 26/31] Fixed YamlRepository serializing plugin zip files as yaml strings --- CHANGELOG.md | 2 + .../Curation/YamlRepositoryTests.cs | 6 ++- Rdmp.Core/Curation/Data/LoadModuleAssembly.cs | 2 + Rdmp.Core/Repositories/YamlRepository.cs | 44 ++++++++++++++++++- .../CommandLine/Gui/ConsoleGuiActivator.cs | 2 +- 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c47a20b537..8c1e4938e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Running RDMP cli without supplying repository connection details (and after deleting `Databases.yaml`) now results in a specific error message instead of null reference [#1346]https://github.com/HicServices/RDMP/issues/1346 - Fixed Pipeline components who run in threaded but call UI methods resulting in unstable UI components [#1357](https://github.com/HicServices/RDMP/issues/1357) +- YamlRepository now saves LoadModuleAssembly binary content as a `.nupkg` file instead of string yaml [#1351](https://github.com/HicServices/RDMP/issues/1351) +- Fixed Console Gui activator 'Select File' dialog having a confusing title of "Directory" [#1282](https://github.com/HicServices/RDMP/issues/1282) ## [7.0.17] - 2022-08-01 diff --git a/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs b/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs index 7d94b8b553..32825fe2b4 100644 --- a/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs +++ b/Rdmp.Core.Tests/Curation/YamlRepositoryTests.cs @@ -282,15 +282,17 @@ public void YamlRepository_LoadSavePluginClass() var lma2 = UnitTests.WhenIHaveA(repo1); - lma1.Plugin.Name = "MyPlugin"; + lma1.Plugin.Name = "MyPlugin1.1.1.1.nupkg"; lma1.Plugin.RdmpVersion = new Version(version); //the version of Rdmp.Core targetted lma1.Plugin.PluginVersion = new Version(1, 1, 1, 1); //the version of the plugin lma1.Plugin.SaveToDatabase(); + lma1.SaveToDatabase(); - lma2.Plugin.Name = "MyPlugin"; + lma2.Plugin.Name = "MyPlugin1.1.1.2.nupkg"; lma2.Plugin.RdmpVersion = new Version(version);//the version of Rdmp.Core targetted (same as above) lma2.Plugin.PluginVersion = new Version(1, 1, 1, 2);//the version of the plugin (higher) lma2.Plugin.SaveToDatabase(); + lma2.SaveToDatabase(); var plugins = repo1.PluginManager.GetCompatiblePlugins(); Assert.That(plugins, Has.Length.EqualTo(1)); diff --git a/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs b/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs index 7756253f3f..82c88a8945 100644 --- a/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs +++ b/Rdmp.Core/Curation/Data/LoadModuleAssembly.cs @@ -16,6 +16,7 @@ using Rdmp.Core.Curation.Data.ImportExport; using Rdmp.Core.Curation.Data.Serialization; using Rdmp.Core.Repositories; +using YamlDotNet.Serialization; namespace Rdmp.Core.Curation.Data { @@ -36,6 +37,7 @@ public class LoadModuleAssembly : DatabaseEntity, IInjectKnown /// /// The assembly (dll) file as a Byte[], use File.WriteAllBytes to write it to disk /// + [YamlIgnore] public Byte[] Bin { get { return _bin;} diff --git a/Rdmp.Core/Repositories/YamlRepository.cs b/Rdmp.Core/Repositories/YamlRepository.cs index 8150464eb2..6cedf60d3f 100644 --- a/Rdmp.Core/Repositories/YamlRepository.cs +++ b/Rdmp.Core/Repositories/YamlRepository.cs @@ -81,7 +81,7 @@ private void LoadObjects() var deserializer = builder.Build(); - foreach (var t in GetCompatibleTypes()) + foreach (var t in GetCompatibleTypes().OrderBy(ObjectDependencyOrder)) { // find the directory that contains all the YAML files e.g. MyDir/Catalogue/ var typeDir = subdirs.FirstOrDefault(d => d.Name.Equals(t.Name)); @@ -120,6 +120,18 @@ private void LoadObjects() LoadWhereSubContainers(); } + private int ObjectDependencyOrder(Type arg) + { + // Load Plugin objects before dependent children + if (arg == typeof(Rdmp.Core.Curation.Data.Plugin)) + return 1; + + if (arg == typeof(LoadModuleAssembly)) + return 2; + + return 3; + } + /// /// Sets on . @@ -138,6 +150,12 @@ protected virtual void SetRepositoryOnObject(IMapsDirectlyToDatabaseTable obj) case ConcreteContainer container: container.SetManager(this); break; + case LoadModuleAssembly lma: + lock(lockFs) + { + lma.Bin = File.ReadAllBytes(GetNupkgPath(lma)); + break; + } } } @@ -150,7 +168,15 @@ public override void InsertAndHydrate(T toCreate, Dictionary { SaveToDatabase(toCreate); } - + } + + private string GetNupkgPath(LoadModuleAssembly lma) + { + //somedir/LoadModuleAssembly/ + var path = Path.GetDirectoryName(GetPath(lma)); + + //somedir/LoadModuleAssembly/MyPlugin1.0.0.nupkg + return Path.Combine(path, GetObjectByID(lma.Plugin_ID).Name); } public override void DeleteFromDatabase(IMapsDirectlyToDatabaseTable oTableWrapperObject) @@ -159,6 +185,12 @@ public override void DeleteFromDatabase(IMapsDirectlyToDatabaseTable oTableWrapp { base.DeleteFromDatabase(oTableWrapperObject); File.Delete(GetPath(oTableWrapperObject)); + + // if deleting a LoadModuleAssembly also delete its binary content file (the plugin dlls in nupkg) + if (oTableWrapperObject is LoadModuleAssembly lma) + { + File.Delete(GetNupkgPath(lma)); + } } } @@ -171,6 +203,14 @@ public override void SaveToDatabase(IMapsDirectlyToDatabaseTable o) lock (lockFs) { File.WriteAllText(GetPath(o), yaml); + + // Do not write plugin binary content into yaml that results in + // a massive blob of binary yaml (not useful and slow to load) + if (o is LoadModuleAssembly lma) + { + // write the nupkg as a binary file instead to the same folder + File.WriteAllBytes(GetNupkgPath(lma), lma.Bin); + } } } diff --git a/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs b/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs index aa3d63e6be..0e081e3b3e 100644 --- a/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs +++ b/Tools/rdmp/CommandLine/Gui/ConsoleGuiActivator.cs @@ -235,7 +235,7 @@ public override FileInfo SelectFile(string prompt) public override FileInfo SelectFile(string prompt, string patternDescription, string pattern) { - var openDir = new OpenDialog(prompt,"Directory") + var openDir = new OpenDialog(prompt,"File") { AllowsMultipleSelection = false, AllowedFileTypes = pattern == null ? null : new []{pattern.TrimStart('*')} From 64f73f9255ad8ee20f59759e35ff0268d2660e04 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 11:45:26 +0100 Subject: [PATCH 27/31] Switch to Spectre.Console for non gui console use --- Plugins/Plugin.Test/Plugin.Test.nuspec | 3 +- Plugins/Plugin.UI/Plugin.UI.nuspec | 3 +- Plugins/Plugin/Plugin.nuspec | 3 +- .../CommandLine/Interactive/AutoComplete.cs | 2 +- .../Interactive/ConsoleInputManager.cs | 149 +++++++++++------- Rdmp.Core/Rdmp.Core.csproj | 2 +- Tools/rdmp/Program.cs | 7 + 7 files changed, 105 insertions(+), 64 deletions(-) diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 2a2111b97a..edd7b02fd2 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -37,8 +37,7 @@ - - + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index e09c26f56e..35187b8b6b 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -40,8 +40,7 @@ - - + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index 70fe228af5..f7cb7b05d2 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -33,8 +33,7 @@ - - + diff --git a/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs b/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs index e6511ccc27..5c2eb4aa56 100644 --- a/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs +++ b/Rdmp.Core/CommandLine/Interactive/AutoComplete.cs @@ -9,7 +9,7 @@ namespace Rdmp.Core.CommandLine.Interactive { - class AutoComplete : IAutoCompleteHandler + class AutoComplete { private readonly string[] autocompletes; diff --git a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs index 30b4a22d69..c16261d33e 100644 --- a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs +++ b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs @@ -10,6 +10,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Text; using FAnsi.Discovery; using MapsDirectlyToDatabaseTable; using Rdmp.Core.CohortCommitting.Pipeline; @@ -27,6 +28,7 @@ using ReusableLibraryCode; using ReusableLibraryCode.Checks; using ReusableLibraryCode.DataAccess; +using Spectre.Console; namespace Rdmp.Core.CommandLine.Interactive { @@ -59,8 +61,7 @@ public override void Show(string title,string message) public override bool TypeText(DialogArgs args, int maxLength, string initialText, out string text, bool requireSaneHeaderText) { - WritePromptFor(args); - text = ReadLineWithAuto(); + text = AnsiConsole.Ask(GetPromptFor(args)); return !string.IsNullOrWhiteSpace(text); } @@ -69,8 +70,7 @@ public override DiscoveredDatabase SelectDatabase(bool allowDatabaseCreation, st if (DisallowInput) throw new InputDisallowedException($"Value required for '{taskDescription}'"); - Console.WriteLine(taskDescription); - var value = ReadLineWithAuto(new PickDatabase()); + var value = ReadLineWithAuto(new DialogArgs { WindowTitle = taskDescription}, new PickDatabase()); return value.Database; } @@ -79,8 +79,7 @@ public override DiscoveredTable SelectTable(bool allowDatabaseCreation, string t if (DisallowInput) throw new InputDisallowedException($"Value required for '{taskDescription}'"); - Console.WriteLine(taskDescription); - var value = ReadLineWithAuto(new PickTable()); + var value = ReadLineWithAuto(new DialogArgs { WindowTitle = taskDescription },new PickTable()); return value.Table; } @@ -133,9 +132,7 @@ public override bool SelectType(DialogArgs args, Type[] available,out Type chose public override IMapsDirectlyToDatabaseTable[] SelectMany(DialogArgs args, Type arrayElementType, IMapsDirectlyToDatabaseTable[] availableObjects) { - WritePromptFor(args); - - var value = ReadLineWithAuto(new PickObjectBase[] + var value = ReadLineWithAuto(args,new PickObjectBase[] {new PickObjectByID(this), new PickObjectByName(this)}, availableObjects.Select(t=>t.GetType().Name).Distinct()); @@ -153,27 +150,35 @@ public override IMapsDirectlyToDatabaseTable[] SelectMany(DialogArgs args, Type /// /// /// Thrown if is true - private void WritePromptFor(DialogArgs args, bool entryLabel = true) + private string GetPromptFor(DialogArgs args, bool entryLabel = true) { if (DisallowInput) throw new InputDisallowedException($"Value required for '{args}'"); + var sb = new StringBuilder(); + if (!string.IsNullOrWhiteSpace(args.WindowTitle)) { - Console.WriteLine(args.WindowTitle); + sb.Append(args.WindowTitle); + + if(entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) + { + sb.Append(" - "); + } } - - if (!string.IsNullOrWhiteSpace(args.TaskDescription)) + + if (entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) { - Console.WriteLine(args.TaskDescription); + sb.Append($"[green]{args.TaskDescription}[/]"); } - - if (entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) + if (!string.IsNullOrWhiteSpace(args.TaskDescription)) { - Console.Write(args.EntryLabel); + sb.AppendLine(); + sb.Append($"[grey]{args.TaskDescription}[/]"); } - + + return sb.ToString(); } public override IMapsDirectlyToDatabaseTable SelectOne(DialogArgs args, IMapsDirectlyToDatabaseTable[] availableObjects) @@ -192,7 +197,7 @@ public override IMapsDirectlyToDatabaseTable SelectOne(DialogArgs args, IMapsDir Console.Write(args.EntryLabel); - var value = ReadLineWithAuto(new PickObjectBase[] + var value = ReadLineWithAuto(args, new PickObjectBase[] {new PickObjectByID(this), new PickObjectByName(this)}, availableObjects.Select(t=>t.GetType().Name).Distinct()); @@ -231,31 +236,29 @@ public override bool SelectObject(DialogArgs args, T[] available, out T selec return false; } - private string ReadLineWithAuto(IEnumerable autoComplete = null) + private string ReadLineWithAuto(DialogArgs args, IEnumerable autoComplete = null) { if (DisallowInput) throw new InputDisallowedException("Value required"); - ReadLine.AutoCompletionHandler = new AutoComplete(autoComplete); - - return ReadLine.Read(); + return AnsiConsole.Ask(GetPromptFor(args)); } - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(PickObjectBase picker) + private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, PickObjectBase picker) { if (DisallowInput) throw new InputDisallowedException("Value required"); - string line = ReadLineWithAuto(picker.GetAutoCompleteIfAny()); + string line = ReadLineWithAuto(args,picker.GetAutoCompleteIfAny()); return picker.Parse(line, 0); } - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(PickObjectBase[] pickers,IEnumerable autoComplete) + private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, PickObjectBase[] pickers,IEnumerable autoComplete) { if (DisallowInput) throw new InputDisallowedException("Value required"); - string line = ReadLineWithAuto(autoComplete); + string line = ReadLineWithAuto(args, autoComplete); var picker = new CommandLineObjectPicker(new[]{line},RepositoryLocator,pickers); return picker[0]; @@ -266,8 +269,19 @@ public override DirectoryInfo SelectDirectory(string prompt) if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - return new DirectoryInfo(Console.ReadLine()); + var result = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select Directory", + EntryLabel = prompt + })) + .AllowEmpty()); + + if (IsBasicallyNull(result)) + return null; + + return new DirectoryInfo(result); } public override FileInfo SelectFile(string prompt) @@ -283,31 +297,52 @@ public override FileInfo SelectFile(string prompt, string patternDescription, st if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - var file = Console.ReadLine(); + var result = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select File", + EntryLabel = prompt + })) + .AllowEmpty()); - // if user types the literal string null then return null (typically interpretted as - 'I don't want to pick a file') - // but not the same as task cancellation - if (string.Equals(file,"null", StringComparison.CurrentCultureIgnoreCase)) + if (IsBasicallyNull(result)) return null; - if (file != null) - return new FileInfo(file); + return new FileInfo(result); + } + + private bool IsBasicallyNull(string result) + { + if (string.IsNullOrWhiteSpace(result)) + return true; + + // if user types the literal string null then return null (typically interpretted as - 'I don't want to pick one') + // but not the same as task cancellation + if (string.Equals(result, "null", StringComparison.CurrentCultureIgnoreCase)) + return true; - return null; + return false; } - + public override FileInfo[] SelectFiles(string prompt, string patternDescription, string pattern) { if (DisallowInput) throw new InputDisallowedException($"Value required for '{prompt}'"); - Console.WriteLine(prompt); - Console.WriteLine(@"Enter path with optional wildcards (e.g. c:\*.csv):"); + var file = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(new DialogArgs + { + WindowTitle = "Select File(s)", + TaskDescription = patternDescription, + EntryLabel = prompt + })) + .AllowEmpty()); - var file = Console.ReadLine(); + if (IsBasicallyNull(file)) + return null; - if (file == null) return null; var asteriskIdx = file.IndexOf('*'); if(asteriskIdx != -1) @@ -337,32 +372,34 @@ public override FileInfo[] SelectFiles(string prompt, string patternDescription, protected override bool SelectValueTypeImpl(DialogArgs args, Type paramType, object initialValue,out object chosen) { - WritePromptFor(args); - - chosen = UsefulStuff.ChangeType(ReadLineWithAuto(), paramType); - + chosen = UsefulStuff.ChangeType(AnsiConsole.Ask(GetPromptFor(args)), paramType); return true; } public override bool YesNo(DialogArgs args, out bool chosen) { - WritePromptFor(args, false); + var result = GetString(args, new List { "Yes","No","Cancel"}); - Console.WriteLine(args.EntryLabel + "(Y/n)"); - //if user picks no then it's false otherwise true - chosen = !string.Equals(Console.ReadLine()?.Trim(), "n", StringComparison.CurrentCultureIgnoreCase); - - //user made a conscious decision - return true; + if (result == "Yes") + chosen = true; + else + chosen = false; + + //user made a noncancel decision? + return result != "Cancel" && !string.IsNullOrWhiteSpace(result); } public string GetString(DialogArgs args, List options) { - WritePromptFor(args); + var chosen = AnsiConsole.Prompt( + new SelectionPrompt() + .PageSize(10) + .Title(GetPromptFor(args)) + .AddChoices(options) + ); - ReadLine.AutoCompletionHandler = new AutoComplete(options); - return ReadLine.Read(); + return chosen; } public override void ShowData(IViewSQLAndResultsCollection collection) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 0cc9a1c817..2c7047ccd9 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -267,7 +267,7 @@ - + diff --git a/Tools/rdmp/Program.cs b/Tools/rdmp/Program.cs index b94cbf90b0..2bffbd1838 100644 --- a/Tools/rdmp/Program.cs +++ b/Tools/rdmp/Program.cs @@ -31,6 +31,12 @@ namespace Rdmp.Core { class Program { + /// + /// True if the user passed the -q switch at startup to suppress any helpful messages we might + /// show (e.g. maybe they want to pipe the results somewhere) + /// + public static bool Quiet { get; private set; } + static int Main(string[] args) { try @@ -51,6 +57,7 @@ static int Main(string[] args) if(args.Any(a=>a.Equals("-q")) || args.Any(a=>a.Equals("--quiet",StringComparison.CurrentCultureIgnoreCase))) { + Quiet = true; foreach(var t in LogManager.Configuration.AllTargets.ToArray()) { if(t.GetType().Name.Contains("Console",StringComparison.CurrentCultureIgnoreCase)) From c14e9f758fa8bab71bde2bc0573ab6bce4b0cda7 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 12:20:32 +0100 Subject: [PATCH 28/31] Improve prompts for Spectre.Console --- .../Interactive/ConsoleInputManager.cs | 67 +++++++++---------- .../Runners/ExecuteCommandRunner.cs | 6 +- 2 files changed, 36 insertions(+), 37 deletions(-) diff --git a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs index c16261d33e..76f1f3f41b 100644 --- a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs +++ b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs @@ -7,24 +7,18 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; using System.IO; using System.Linq; using System.Text; using FAnsi.Discovery; using MapsDirectlyToDatabaseTable; -using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandLine.Interactive.Picking; -using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Aggregation; using Rdmp.Core.Curation.Data.DataLoad; -using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataExport.DataExtraction; using Rdmp.Core.DataViewing; -using Rdmp.Core.Logging; using Rdmp.Core.Repositories; -using Rdmp.Core.Startup; using ReusableLibraryCode; using ReusableLibraryCode.Checks; using ReusableLibraryCode.DataAccess; @@ -133,8 +127,7 @@ public override IMapsDirectlyToDatabaseTable[] SelectMany(DialogArgs args, Type IMapsDirectlyToDatabaseTable[] availableObjects) { var value = ReadLineWithAuto(args,new PickObjectBase[] - {new PickObjectByID(this), new PickObjectByName(this)}, - availableObjects.Select(t=>t.GetType().Name).Distinct()); + {new PickObjectByID(this), new PickObjectByName(this)}); var unavailable = value.DatabaseEntities.Except(availableObjects).ToArray(); @@ -149,8 +142,9 @@ public override IMapsDirectlyToDatabaseTable[] SelectMany(DialogArgs args, Type /// /// /// + /// /// Thrown if is true - private string GetPromptFor(DialogArgs args, bool entryLabel = true) + private string GetPromptFor(DialogArgs args, bool entryLabel = true, params PickObjectBase[] pickers) { if (DisallowInput) throw new InputDisallowedException($"Value required for '{args}'"); @@ -159,7 +153,7 @@ private string GetPromptFor(DialogArgs args, bool entryLabel = true) if (!string.IsNullOrWhiteSpace(args.WindowTitle)) { - sb.Append(args.WindowTitle); + sb.Append(Markup.Escape(args.WindowTitle)); if(entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) { @@ -169,13 +163,33 @@ private string GetPromptFor(DialogArgs args, bool entryLabel = true) if (entryLabel && !string.IsNullOrWhiteSpace(args.EntryLabel)) { - sb.Append($"[green]{args.TaskDescription}[/]"); + sb.Append($"[green]{Markup.Escape(args.EntryLabel)}[/]"); } if (!string.IsNullOrWhiteSpace(args.TaskDescription)) { sb.AppendLine(); - sb.Append($"[grey]{args.TaskDescription}[/]"); + sb.Append($"[grey]{Markup.Escape(args.TaskDescription)}[/]"); + } + + foreach(var picker in pickers) + { + sb.AppendLine(); + sb.Append($"Format:[grey]{Markup.Escape(picker.Format)}[/]"); + + if(picker.Examples.Any()) + { + + sb.AppendLine(); + sb.Append($"Examples:"); + foreach (var example in picker.Examples) + { + sb.AppendLine(); + sb.Append($"[grey]{Markup.Escape(example)}[/]"); + } + } + sb.AppendLine(); + sb.Append(":"); } return sb.ToString(); @@ -198,8 +212,7 @@ public override IMapsDirectlyToDatabaseTable SelectOne(DialogArgs args, IMapsDir Console.Write(args.EntryLabel); var value = ReadLineWithAuto(args, new PickObjectBase[] - {new PickObjectByID(this), new PickObjectByName(this)}, - availableObjects.Select(t=>t.GetType().Name).Distinct()); + {new PickObjectByID(this), new PickObjectByName(this)}); var chosen = value.DatabaseEntities?.SingleOrDefault(); @@ -236,32 +249,18 @@ public override bool SelectObject(DialogArgs args, T[] available, out T selec return false; } - private string ReadLineWithAuto(DialogArgs args, IEnumerable autoComplete = null) + private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, params PickObjectBase[] pickers) { if (DisallowInput) throw new InputDisallowedException("Value required"); - return AnsiConsole.Ask(GetPromptFor(args)); - } - - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, PickObjectBase picker) - { - if (DisallowInput) - throw new InputDisallowedException("Value required"); - - string line = ReadLineWithAuto(args,picker.GetAutoCompleteIfAny()); + var line = AnsiConsole.Prompt( + new TextPrompt( + GetPromptFor(args,true, pickers).Trim())); - return picker.Parse(line, 0); - } - private CommandLineObjectPickerArgumentValue ReadLineWithAuto(DialogArgs args, PickObjectBase[] pickers,IEnumerable autoComplete) - { - if (DisallowInput) - throw new InputDisallowedException("Value required"); - string line = ReadLineWithAuto(args, autoComplete); - - var picker = new CommandLineObjectPicker(new[]{line},RepositoryLocator,pickers); - return picker[0]; + var cli = new CommandLineObjectPicker(new[] { line }, RepositoryLocator, pickers); + return cli[0]; } public override DirectoryInfo SelectDirectory(string prompt) diff --git a/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs b/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs index ee0aa06aaf..fabddb28f3 100644 --- a/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs +++ b/Rdmp.Core/CommandLine/Runners/ExecuteCommandRunner.cs @@ -20,6 +20,7 @@ using Rdmp.Core.Repositories; using ReusableLibraryCode.Checks; using ReusableLibraryCode.Progress; +using Spectre.Console; namespace Rdmp.Core.CommandLine.Runners { @@ -138,8 +139,7 @@ private void RunCommandExecutionLoop(IRDMPPlatformRepositoryServiceLocator repos while (true) { - Console.WriteLine("Enter Command (or 'exit')"); - var command = _input.GetString(new DialogArgs { WindowTitle = "Command" }, _commands.Keys.ToList()); + var command = _input.GetString(new DialogArgs { WindowTitle = "Enter Command (or Ctrl+C)" }, _commands.Keys.ToList()); try { command = GetCommandAndPickerFromLine(command, out _picker,repositoryLocator); @@ -151,7 +151,7 @@ private void RunCommandExecutionLoop(IRDMPPlatformRepositoryServiceLocator repos } catch (Exception ex) { - Console.WriteLine(ex.Message); + AnsiConsole.WriteException(ex, ExceptionFormats.ShortenEverything); } From 0123027e03b3e21b02962685ff6c2aab918c589b Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 12:28:58 +0100 Subject: [PATCH 29/31] Added Spectre.Console to packages markdown and nuspec --- Documentation/CodeTutorials/Packages.md | 2 +- Plugins/Plugin.Test/Plugin.Test.nuspec | 1 + Plugins/Plugin.UI/Plugin.UI.nuspec | 1 + Plugins/Plugin/Plugin.nuspec | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index bb2d8d39f6..aee1f0cdfa 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -45,7 +45,7 @@ | System.Security.Permissions |[GitHub](https://github.com/dotnet/corefx) | [6.0.0](https://www.nuget.org/packages/System.Security.Permissions/6.0.0) |[MIT](https://opensource.org/licenses/MIT) | Provides common types for Xml doc reading in UI code | | | [AutoComplete Console](https://www.codeproject.com/Articles/1182358/Using-Autocomplete-in-Windows-Console-Applications) by Jasper Lammers | Embedded | 4.0 | [CPOL](https://www.codeproject.com/info/cpol10.aspx) | Provides interactive autocomplete in console input | | | System.Resources.Extensions | [GitHub](https://github.com/dotnet/corefx) | [4.6.0](https://www.nuget.org/packages/System.Resources.Extensions/4.6.0) | [MIT](https://opensource.org/licenses/MIT) | Allows [publishing with dotnet publish on machines with netcoreapp3.0 SDK installed](https://github.com/microsoft/msbuild/issues/4704#issuecomment-530034240) | | -| ReadLine | [GitHub](https://github.com/tonerdo/readline) | [2.0.1](https://www.nuget.org/packages/ReadLine/2.0.1) | [MIT](https://opensource.org/licenses/MIT) | Allows autocomplete on command line | | +| Spectre.Console | [GitHub](https://github.com/spectreconsole/spectre.console) | [0.44.0](https://www.nuget.org/packages/Spectre.Console/0.44.0) | [MIT](https://opensource.org/licenses/MIT) | Allows richer command line interactions| | | HIC.System.Windows.Forms.DataVisualization | [GitHub](https://github.com/HicServices/winforms-datavisualization) | [1.0.1](https://www.nuget.org/packages/HIC.System.Windows.Forms.DataVisualization/1.0.1) |[MIT](https://opensource.org/licenses/MIT) | Dotnet core support for DQE charts | | | System.DirectoryServices.Protocols | [GitHub](https://github.com/dotnet/runtime) | [6.0.1](https://www.nuget.org/packages/System.DirectoryServices.Protocols/6.0.1) | MIT | Required dependency of Oracle when using LDAP auth | | Autoupdater.NET | [GitHub](https://github.com/ravibpatel/AutoUpdater.NET) | [1.7.0](https://github.com/ravibpatel/AutoUpdater.NET) | MIT | Manages updating of the RDMP windows client directly from the RDMP GitHub Releases| diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index edd7b02fd2..f0c3c27967 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -18,6 +18,7 @@ + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index 35187b8b6b..80ba49f04d 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -22,6 +22,7 @@ + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index f7cb7b05d2..47b3f249cb 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -16,6 +16,7 @@ + From 9fd66a18c481698c021af80de057ab706c4c9b09 Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 12:51:08 +0100 Subject: [PATCH 30/31] Allow empty values to be typed on console for TypeText --- .../Interactive/ConsoleInputManager.cs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs index 76f1f3f41b..713fbaa6a6 100644 --- a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs +++ b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs @@ -55,8 +55,23 @@ public override void Show(string title,string message) public override bool TypeText(DialogArgs args, int maxLength, string initialText, out string text, bool requireSaneHeaderText) { - text = AnsiConsole.Ask(GetPromptFor(args)); - return !string.IsNullOrWhiteSpace(text); + text = AnsiConsole.Prompt( + new TextPrompt(GetPromptFor(args)) + .AllowEmpty() + ); + + if(string.Equals(text , "Cancel",StringComparison.CurrentCultureIgnoreCase)) + { + // user does not want to type any text + return false; + } + + // user typed "null" or some spaces or something + if (IsBasicallyNull(text)) + text = null; + + // thats still an affirmative choice + return true; } public override DiscoveredDatabase SelectDatabase(bool allowDatabaseCreation, string taskDescription) From 8535f8f568faa82c551b8acd8b035a7302e1008d Mon Sep 17 00:00:00 2001 From: tznind Date: Tue, 30 Aug 2022 13:00:24 +0100 Subject: [PATCH 31/31] Added ConsoleInputManager implementation of Wait --- .../CommandLine/Interactive/ConsoleInputManager.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs index 713fbaa6a6..96dea5c880 100644 --- a/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs +++ b/Rdmp.Core/CommandLine/Interactive/ConsoleInputManager.cs @@ -10,6 +10,8 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using FAnsi.Discovery; using MapsDirectlyToDatabaseTable; using Rdmp.Core.CommandExecution; @@ -503,5 +505,14 @@ public override void LaunchSubprocess(ProcessStartInfo startInfo) { throw new NotSupportedException(); } + + public override void Wait(string title, Task task, CancellationTokenSource cts) + { + AnsiConsole.Status() + .Spinner(Spinner.Known.Star) + .Start(title, ctx => + base.Wait(title, task, cts) + ); + } } }