From b1e049ed8c95b14abe1289d741d95a3428ab6f73 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Thu, 9 Jun 2022 13:35:09 +0100 Subject: [PATCH 01/20] Adjust number of Catalogues and Projects to match DLS --- .../ExampleDatasetsCreation.cs | 25 +--- .../DatabaseCreation/NightmareDatasets.cs | 109 ++++++++++++++++++ 2 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs index 8acdc41cfb..13f3758b1e 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs @@ -203,29 +203,8 @@ WHERE Operation <> 'NULL' AND Operation <> 'Nul' if (options.Nightmare) { - //Lots of datasets - for (int i = 0; i < 1000; i++) - { - var cata = new Catalogue(_repos.CatalogueRepository, $"Catalogue {i}"); - var eds = new ExtractableDataSet(_repos.DataExportRepository, cata); - var ti = new TableInfo(_repos.CatalogueRepository, $"[MyDb].[Table{i}]"); - - for (int j = 0; j < 40; j++) - { - var col = new ColumnInfo(_repos.CatalogueRepository, $"MyCol{j}", "varchar(10)", ti); - var ci = new CatalogueItem(_repos.CatalogueRepository, cata, col.Name); - new ExtractionInformation(_repos.CatalogueRepository, ci, col, col.Name); - } - - Project p = new Project(_repos.DataExportRepository,$"Project {i}"); - - for (int j = 0; j < 20; j++) - { - var config = new ExtractionConfiguration(_repos.DataExportRepository, p); - new SelectedDataSets(_repos.DataExportRepository,config, eds,null); - } - } - + var nightmare = new NightmareDatasets(_repos); + nightmare.Create(); } } diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs new file mode 100644 index 0000000000..4489b55168 --- /dev/null +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -0,0 +1,109 @@ +// Copyright (c) The University of Dundee 2018-2019 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using BadMedicine; +using BadMedicine.Datasets; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Repositories; +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Rdmp.Core.CommandLine.DatabaseCreation +{ + internal class NightmareDatasets : DataGenerator + { + private IRDMPPlatformRepositoryServiceLocator _repos; + + public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos):base(new Random(123)) + { + _repos = repos; + } + + /// + /// Generates a lot of metadata in the RDMP platform databases. This is for testing + /// system scalability. + /// + /// We use a lot, this is just because it is a nice + /// short string of letter and numbers not because we are actually using GP codes + /// + [SuppressMessage("Security", "SCS0005:Weak random number generator.", Justification = "We are generating random metadata, security does not enter into the equation")] + public void Create() + { + var catalogues = new BucketList(); + var extractableDatasets = new BucketList(); + + // Based on DLS figures see: https://github.com/HicServices/RDMP/issues/1224 + for (int i = 0; i < 1000; i++) + { + var cata = new Catalogue(_repos.CatalogueRepository, $"Catalogue {GetRandomGPCode(r)}"); + cata.Description = GetRandomSentence(r); + cata.SaveToDatabase(); + catalogues.Add(1,cata); + + var ti = new TableInfo(_repos.CatalogueRepository, $"[MyDb].[Table{i}]"); + + // half of datasets have linkage identifiers + bool hasExtractionIdentifier = r.Next(2) == 0; + + for (int j = 0; j < 40; j++) + { + var col = new ColumnInfo(_repos.CatalogueRepository, $"MyCol{j}", "varchar(10)", ti); + var ci = new CatalogueItem(_repos.CatalogueRepository, cata, col.Name); + var ei = new ExtractionInformation(_repos.CatalogueRepository, ci, col, col.Name); + + // make the first field the linkage identifier + // if we are doing that + if (j == 0 && hasExtractionIdentifier) + { + ei.IsExtractionIdentifier = true; + ei.SaveToDatabase(); + } + } + + // half of the Catalogues have IsExtractionIdentifier + // but lets make only 75% of those extractable datasets + if (r.Next(5) > 0 && hasExtractionIdentifier) + { + var eds = new ExtractableDataSet(_repos.DataExportRepository, cata); + extractableDatasets.Add(1, eds); + } + } + + + for (int i = 0; i < 200; i++) + { + // each project + Project p = new Project(_repos.DataExportRepository, $"Project {i}"); + + // has an average of 5 ExtractionConfigurations but could have 0 to 10 + var configs = GetGaussianInt(0, 10); + + for(int c = 0;c Date: Thu, 9 Jun 2022 14:06:59 +0100 Subject: [PATCH 02/20] Nightmare support for cic and proj specific --- .../DatabaseCreation/NightmareDatasets.cs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index 4489b55168..06be2636d9 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -7,6 +7,7 @@ using BadMedicine; using BadMedicine.Datasets; using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Repositories; using System; @@ -35,6 +36,7 @@ public void Create() { var catalogues = new BucketList(); var extractableDatasets = new BucketList(); + var projects = new BucketList(); // Based on DLS figures see: https://github.com/HicServices/RDMP/issues/1224 for (int i = 0; i < 1000; i++) @@ -78,11 +80,12 @@ public void Create() { // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); + projects.Add(1, p); // has an average of 5 ExtractionConfigurations but could have 0 to 10 - var configs = GetGaussianInt(0, 10); + var numberOfConfigs = GetGaussianInt(0, 10); - for(int c = 0;c Date: Thu, 9 Jun 2022 14:46:25 +0100 Subject: [PATCH 03/20] Not extractable TableInfos and vary CatalogueItem extractabilities --- .../ExampleDatasetsCreation.cs | 2 +- .../DatabaseCreation/NightmareDatasets.cs | 120 ++++++++++++++---- 2 files changed, 97 insertions(+), 25 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs index 13f3758b1e..10f59259af 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs @@ -203,7 +203,7 @@ WHERE Operation <> 'NULL' AND Operation <> 'Nul' if (options.Nightmare) { - var nightmare = new NightmareDatasets(_repos); + var nightmare = new NightmareDatasets(_repos,db); nightmare.Create(); } } diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index 06be2636d9..1ea7e0094b 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -6,24 +6,45 @@ using BadMedicine; using BadMedicine.Datasets; +using FAnsi.Discovery; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; using Rdmp.Core.Repositories; using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; namespace Rdmp.Core.CommandLine.DatabaseCreation { + [SuppressMessage("Security", "SCS0005:Weak random number generator.", Justification = "We are generating random metadata, security does not enter into the equation")] internal class NightmareDatasets : DataGenerator { private IRDMPPlatformRepositoryServiceLocator _repos; - - public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos):base(new Random(123)) + private DiscoveredDatabase _db; + private string _serverName; + private string _databaseNameWrapped; + private string _databaseNameRuntime; + + public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos, DiscoveredDatabase db) :base(new Random(123)) { _repos = repos; + _db = db; + _serverName = _db.Server.Name; + _databaseNameWrapped = _db.GetWrappedName(); + _databaseNameRuntime = _db.GetRuntimeName(); } + BucketList Catalogues = new (); + BucketList ExtractableDatasets = new (); + BucketList Projects = new (); + BucketList Tables = new (); + int TablesCount = 0; + + BucketList Columns = new(); + int ColumnsCount = 0; + /// /// Generates a lot of metadata in the RDMP platform databases. This is for testing /// system scalability. @@ -31,39 +52,53 @@ public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos):base(new R /// We use a lot, this is just because it is a nice /// short string of letter and numbers not because we are actually using GP codes /// - [SuppressMessage("Security", "SCS0005:Weak random number generator.", Justification = "We are generating random metadata, security does not enter into the equation")] public void Create() { - var catalogues = new BucketList(); - var extractableDatasets = new BucketList(); - var projects = new BucketList(); + // how likely is a given ExtractionInformation to be each of these + // categories + var extractionCategories = new BucketList(); + extractionCategories.Add(100, ExtractionCategory.Core); + extractionCategories.Add(10, ExtractionCategory.Internal); + extractionCategories.Add(5, ExtractionCategory.Supplemental); + extractionCategories.Add(10, ExtractionCategory.SpecialApprovalRequired); + extractionCategories.Add(4, ExtractionCategory.Deprecated); + // Based on DLS figures see: https://github.com/HicServices/RDMP/issues/1224 - for (int i = 0; i < 1000; i++) + for (int i = 0; i < 500; i++) { var cata = new Catalogue(_repos.CatalogueRepository, $"Catalogue {GetRandomGPCode(r)}"); cata.Description = GetRandomSentence(r); cata.SaveToDatabase(); - catalogues.Add(1,cata); - - var ti = new TableInfo(_repos.CatalogueRepository, $"[MyDb].[Table{i}]"); + Catalogues.Add(1,cata); // half of datasets have linkage identifiers bool hasExtractionIdentifier = r.Next(2) == 0; + bool first = true; + + // 14497 CatalogueItem + // 8922 ExtractionInformation + // = 60% of columns are extractable - for (int j = 0; j < 40; j++) + foreach (var col in CreateTable()) { - var col = new ColumnInfo(_repos.CatalogueRepository, $"MyCol{j}", "varchar(10)", ti); var ci = new CatalogueItem(_repos.CatalogueRepository, cata, col.Name); - var ei = new ExtractionInformation(_repos.CatalogueRepository, ci, col, col.Name); - // make the first field the linkage identifier - // if we are doing that - if (j == 0 && hasExtractionIdentifier) + // = 60% of columns are extractable + if (r.Next(10) < 6) { - ei.IsExtractionIdentifier = true; - ei.SaveToDatabase(); - } + var ei = new ExtractionInformation(_repos.CatalogueRepository, ci, col, col.Name); + + // make the first field the linkage identifier + // if we are doing that + if (first && hasExtractionIdentifier) + { + ei.ExtractionCategory = extractionCategories.GetRandom(r); + ei.IsExtractionIdentifier = true; + ei.SaveToDatabase(); + first = false; + } + } } // half of the Catalogues have IsExtractionIdentifier @@ -71,16 +106,23 @@ public void Create() if (r.Next(5) > 0 && hasExtractionIdentifier) { var eds = new ExtractableDataSet(_repos.DataExportRepository, cata); - extractableDatasets.Add(1, eds); + ExtractableDatasets.Add(1, eds); } } + // There are 500 tables associated with Catalogues + // but also 250 tables that are not linked to any Catalogues + for (int i = 0; i < 250; i++) + { + CreateTable(); + } + for (int i = 0; i < 200; i++) { // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); - projects.Add(1, p); + Projects.Add(1, p); // has an average of 5 ExtractionConfigurations but could have 0 to 10 var numberOfConfigs = GetGaussianInt(0, 10); @@ -94,12 +136,11 @@ public void Create() config.ReleaseTicket = GetRandomGPCode(r); // some have release tickets config.SaveToDatabase(); - // average of 8 Catalogues per extraction var numberOfsds = GetGaussianInt(0, 16); for (int s = 0; s < numberOfsds; s++) { - var sds = new SelectedDataSets(_repos.DataExportRepository, config,extractableDatasets.GetRandom(r), null); + var sds = new SelectedDataSets(_repos.DataExportRepository, config,ExtractableDatasets.GetRandom(r), null); } @@ -114,12 +155,43 @@ public void Create() // 25% of cics are associated with a specific project if(r.Next(4) == 0) { - var projSpecific = new ProjectCohortIdentificationConfigurationAssociation(_repos.DataExportRepository, projects.GetRandom(r), cic); + var projSpecific = new ProjectCohortIdentificationConfigurationAssociation(_repos.DataExportRepository, Projects.GetRandom(r), cic); } } } + private IEnumerable CreateTable() + { + // 762 tables + // 18415 columns + // = average of 24 columns per table + var ti = new TableInfo(_repos.CatalogueRepository, $"[MyDb].[Table{TablesCount++}]"); + + // lets not set the server name on 1 in 20 so we get all those + // horrible null references out in the open + if(r.Next(20)!=0) + ti.Server = _serverName; + + // lets not set the database name on 1 in 20 so we get all those + // horrible null references out in the open + if (r.Next(20) != 0) + ti.Database = r.Next(2) == 0 ? _databaseNameRuntime : _databaseNameWrapped; // some are "[mydb]" some will be "mydb" + + // 1 in 20 tables is a view + ti.IsView = r.Next(20) == 0; + + ti.SaveToDatabase(); + + Tables.Add(1, ti); + + var numberOfColumns = GetGaussianInt(1, 48); + for (int j = 0; j < numberOfColumns; j++) + { + yield return new ColumnInfo(_repos.CatalogueRepository, $"MyCol{ColumnsCount++}", "varchar(10)", ti); + } + } + // we are not actually interested in these methods, just want to use GetGaussian etc public override object[] GenerateTestDataRow(Person p) { From ec9e6bd95d1cadd764153f062a0c00ad608b59f2 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Thu, 9 Jun 2022 15:38:16 +0100 Subject: [PATCH 04/20] Added `--nightmarefactor` and made Nightmare auto imply ExampleDatasets --- .../DatabaseCreation/ExampleDatasetsCreation.cs | 5 ++++- .../DatabaseCreation/NightmareDatasets.cs | 14 ++++++++++---- .../DatabaseCreation/PlatformDatabaseCreation.cs | 2 +- .../PlatformDatabaseCreationOptions.cs | 5 ++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs index 10f59259af..af299bccf9 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs @@ -203,7 +203,10 @@ WHERE Operation <> 'NULL' AND Operation <> 'Nul' if (options.Nightmare) { - var nightmare = new NightmareDatasets(_repos,db); + var nightmare = new NightmareDatasets(_repos, db) + { + Factor = options.NightmareFactor + }; nightmare.Create(); } } diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index 1ea7e0094b..ad6256aade 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -27,6 +27,12 @@ internal class NightmareDatasets : DataGenerator private string _databaseNameWrapped; private string _databaseNameRuntime; + /// + /// Defaults to 1, set to 2 to double the amount of objects generated. + /// Set to 10 to produce 10 times the amount etc + /// + public int Factor = 1; + public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos, DiscoveredDatabase db) :base(new Random(123)) { _repos = repos; @@ -65,7 +71,7 @@ public void Create() // Based on DLS figures see: https://github.com/HicServices/RDMP/issues/1224 - for (int i = 0; i < 500; i++) + for (int i = 0; i < 500 * Factor; i++) { var cata = new Catalogue(_repos.CatalogueRepository, $"Catalogue {GetRandomGPCode(r)}"); cata.Description = GetRandomSentence(r); @@ -112,13 +118,13 @@ public void Create() // There are 500 tables associated with Catalogues // but also 250 tables that are not linked to any Catalogues - for (int i = 0; i < 250; i++) + for (int i = 0; i < 250 * Factor; i++) { CreateTable(); } - for (int i = 0; i < 200; i++) + for (int i = 0; i < 200 * Factor; i++) { // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); @@ -148,7 +154,7 @@ public void Create() } // 200 cics - for (int i = 0; i < 200; i++) + for (int i = 0; i < 200 * Factor; i++) { var cic = new CohortIdentificationConfiguration(_repos.CatalogueRepository, "Cohort Query " + GetRandomGPCode(r)); diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs index dca306fb50..72a9af2e1a 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreation.cs @@ -50,7 +50,7 @@ public void CreatePlatformDatabases(PlatformDatabaseCreationOptions options) creator.Create(); } - if(options.ExampleDatasets) + if(options.ExampleDatasets || options.Nightmare) { var examples = new ExampleDatasetsCreation(new ThrowImmediatelyActivator(repo,null),repo); var server = new DiscoveredServer(options.GetBuilder("ExampleData")); diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs index ebc95f080e..8460e106dd 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs @@ -50,9 +50,12 @@ public class PlatformDatabaseCreationOptions [Option("NumberOfRowsPerDataset", Default = ExampleDatasetsCreation.NumberOfRowsPerDataset, HelpText = "When ExampleDatasets is set this is the number of rows to create in each dataset")] public int NumberOfRowsPerDataset { get; set; } = ExampleDatasetsCreation.NumberOfRowsPerDataset; - [Option("Nightmare", Default = false, HelpText = "Only applies when using ExampleDatasets. Create a 100,000+ objects in the Catalogue database")] + [Option("Nightmare", Default = false, HelpText = "Create a 100,000+ objects in the Catalogue database")] public bool Nightmare { get; set; } + [Option("NightmareFactor", Default = 1, HelpText = "Set to 2 (or more) to double the volume of Nightmare data generated")] + public int NightmareFactor { get; set; } = 1; + [Option(Required = false, Default = false, HelpText = "Set to true to validate the SSL certificate of the server you are installing into")] public bool ValidateCertificate {get;set;} From f26eafd1fa1b688f95e074ef149d78495c15e82d Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 13 Jun 2022 10:02:56 +0100 Subject: [PATCH 05/20] Nightmare modelling of columns and extraction containers --- .../DatabaseCreation/NightmareDatasets.cs | 43 +++++++++++++++++-- .../Data/ExtractionConfiguration.cs | 15 ++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index ad6256aade..75daee9fe7 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -146,10 +146,30 @@ public void Create() var numberOfsds = GetGaussianInt(0, 16); for (int s = 0; s < numberOfsds; s++) { - var sds = new SelectedDataSets(_repos.DataExportRepository, config,ExtractableDatasets.GetRandom(r), null); - } - + var ds = ExtractableDatasets.GetRandom(r); + config.AddDatasetToConfiguration(ds, out var sds); + + + // FilterContainer 4370 + // most selected datasets have filters + // just add a single root container + if (sds.RootFilterContainer_ID == null) + { + sds.CreateRootContainerIfNotExists(); + AddExtractionFiltersTo(sds.RootFilterContainer); + } + + // FilterContainerSubcontainers 271 + + // 5% have subcontainers + if (r.Next(20) == 0) + { + var subContainer = new FilterContainer(); + AddExtractionFiltersTo(subContainer); + sds.RootFilterContainer.AddChild(subContainer); + } + } } } @@ -167,6 +187,23 @@ public void Create() } } + private void AddExtractionFiltersTo(IContainer container) + { + + var numberOfFilters = GetGaussianInt(0, 2); + for (int f = 0; f < numberOfFilters; f++) + { + var filter = new DeployedExtractionFilter(_repos.DataExportRepository, + $"Filter {Guid.NewGuid()}", null) + { + WhereSQL = "ColX > 0" + }; + filter.SaveToDatabase(); + + container.AddChild(filter); + } + } + private IEnumerable CreateTable() { // 762 tables diff --git a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs index 3fdbf17532..50e78fcf1d 100644 --- a/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs +++ b/Rdmp.Core/DataExport/Data/ExtractionConfiguration.cs @@ -585,13 +585,26 @@ public IExtractableDataSet[] GetAllExtractableDataSets() /// public void AddDatasetToConfiguration(IExtractableDataSet extractableDataSet) { + AddDatasetToConfiguration(extractableDataSet, out _); + } + + /// + /// Makes the provided extractable in the current . This + /// includes selecting it () and replicating any mandatory filters. + /// + /// + /// The RDMP object that indicates that the dataset is extracted in this configuration + public void AddDatasetToConfiguration(IExtractableDataSet extractableDataSet, out ISelectedDataSets selectedDataSet) + { + selectedDataSet = null; + //it is already part of the configuration if( SelectedDataSets.Any(s => s.ExtractableDataSet_ID == extractableDataSet.ID)) return; var dataExportRepo = (IDataExportRepository)Repository; - var selectedDataSet = new SelectedDataSets(dataExportRepo, this, extractableDataSet, null); + selectedDataSet = new SelectedDataSets(dataExportRepo, this, extractableDataSet, null); ExtractionFilter[] mandatoryExtractionFiltersToApplyToDataset = extractableDataSet.Catalogue.GetAllMandatoryFilters(); From 260372d6bc75284c020b5d49686a1c689416a787 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 13 Jun 2022 10:23:50 +0100 Subject: [PATCH 06/20] Fixed filter creation in new nightmare datasets implementation --- .../CommandLine/DatabaseCreation/NightmareDatasets.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index 75daee9fe7..e8f08968b7 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -153,6 +153,10 @@ public void Create() // FilterContainer 4370 // most selected datasets have filters + // we picked the same dataset twice, oh well + if (sds == null) + continue; + // just add a single root container if (sds.RootFilterContainer_ID == null) { @@ -165,9 +169,9 @@ public void Create() // 5% have subcontainers if (r.Next(20) == 0) { - var subContainer = new FilterContainer(); - AddExtractionFiltersTo(subContainer); + var subContainer = new FilterContainer(_repos.DataExportRepository); sds.RootFilterContainer.AddChild(subContainer); + AddExtractionFiltersTo(subContainer); } } } From eaf8d817729153df8b0a4f911bedb76a83f4ebdb Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Mon, 13 Jun 2022 13:22:32 +0100 Subject: [PATCH 07/20] Nightmare support for ExtractableCohorts --- .../DatabaseCreation/ExampleDatasetsCreation.cs | 2 +- .../DatabaseCreation/NightmareDatasets.cs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs index af299bccf9..447726cf80 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/ExampleDatasetsCreation.cs @@ -207,7 +207,7 @@ WHERE Operation <> 'NULL' AND Operation <> 'Nul' { Factor = options.NightmareFactor }; - nightmare.Create(); + nightmare.Create(externalCohortTable); } } diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index e8f08968b7..f02a9222f0 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -7,6 +7,7 @@ using BadMedicine; using BadMedicine.Datasets; using FAnsi.Discovery; +using Rdmp.Core.CohortCommitting.Pipeline; using Rdmp.Core.Curation.Data; using Rdmp.Core.Curation.Data.Cohort; using Rdmp.Core.DataExport.Data; @@ -58,7 +59,7 @@ public NightmareDatasets(IRDMPPlatformRepositoryServiceLocator repos, Discovered /// We use a lot, this is just because it is a nice /// short string of letter and numbers not because we are actually using GP codes /// - public void Create() + public void Create(ExternalCohortTable ect) { // how likely is a given ExtractionInformation to be each of these // categories @@ -123,11 +124,15 @@ public void Create() CreateTable(); } + // open a connection to the cohort db for creating external cohorts + using var con = ect.Discover().Server.GetManagedConnection(); for (int i = 0; i < 200 * Factor; i++) { // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); + p.ProjectNumber = r.Next(50) == 0 ? 5:i; // its ok for some projects to have the same number + p.SaveToDatabase(); Projects.Add(1, p); // has an average of 5 ExtractionConfigurations but could have 0 to 10 @@ -149,6 +154,16 @@ public void Create() var ds = ExtractableDatasets.GetRandom(r); config.AddDatasetToConfiguration(ds, out var sds); + var request = new CohortCreationRequest(p, + new CohortDefinition(null,"Created by NightmareDatasets",1, p.ProjectNumber.Value, ect), + _repos.DataExportRepository, + $"Nightmare dreamed on {DateTime.Now}"); + + request.PushToServer(con); + + var cohort = new ExtractableCohort(_repos.DataExportRepository, ect, request.NewCohortDefinition.ID.Value); + config.Cohort_ID = cohort.ID; + config.SaveToDatabase(); // FilterContainer 4370 // most selected datasets have filters From dca89eb35b2e1696135ea123d0f502358c7bf6c3 Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Wed, 15 Jun 2022 07:33:47 +0100 Subject: [PATCH 08/20] Fixed Nightmare pick sds that have no extraction identifier and Project having no set extraction dir --- .../DatabaseCreation/NightmareDatasets.cs | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index f02a9222f0..92d446684d 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; namespace Rdmp.Core.CommandLine.DatabaseCreation @@ -70,6 +71,8 @@ public void Create(ExternalCohortTable ect) extractionCategories.Add(10, ExtractionCategory.SpecialApprovalRequired); extractionCategories.Add(4, ExtractionCategory.Deprecated); + var extractionDir = Path.GetTempPath(); + // Based on DLS figures see: https://github.com/HicServices/RDMP/issues/1224 for (int i = 0; i < 500 * Factor; i++) @@ -80,7 +83,7 @@ public void Create(ExternalCohortTable ect) Catalogues.Add(1,cata); // half of datasets have linkage identifiers - bool hasExtractionIdentifier = r.Next(2) == 0; + bool hasExtractionIdentifier = false; bool first = true; // 14497 CatalogueItem @@ -95,16 +98,23 @@ public void Create(ExternalCohortTable ect) if (r.Next(10) < 6) { var ei = new ExtractionInformation(_repos.CatalogueRepository, ci, col, col.Name); + ei.ExtractionCategory = extractionCategories.GetRandom(r); - // make the first field the linkage identifier - // if we are doing that - if (first && hasExtractionIdentifier) + if (first) { - ei.ExtractionCategory = extractionCategories.GetRandom(r); - ei.IsExtractionIdentifier = true; - ei.SaveToDatabase(); - first = false; + hasExtractionIdentifier = r.Next(2) == 0; + + // make the first field the linkage identifier + // if we are doing that + if (hasExtractionIdentifier) + { + ei.IsExtractionIdentifier = true; + ei.ExtractionCategory = ExtractionCategory.Core; + ei.SaveToDatabase(); + } } + + first = false; } } @@ -132,6 +142,7 @@ public void Create(ExternalCohortTable ect) // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); p.ProjectNumber = r.Next(50) == 0 ? 5:i; // its ok for some projects to have the same number + p.ExtractionDirectory = extractionDir; p.SaveToDatabase(); Projects.Add(1, p); From ba6f584b2898bc089bbde90911fe84ae777b0d04 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Thu, 21 Jul 2022 09:04:10 -0500 Subject: [PATCH 09/20] Typo/text fixes --- Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs | 5 ++--- .../DatabaseCreation/PlatformDatabaseCreationOptions.cs | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs index 92d446684d..f80b5a250d 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/NightmareDatasets.cs @@ -16,7 +16,6 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; -using System.Linq; namespace Rdmp.Core.CommandLine.DatabaseCreation { @@ -141,7 +140,7 @@ public void Create(ExternalCohortTable ect) { // each project Project p = new Project(_repos.DataExportRepository, $"Project {i}"); - p.ProjectNumber = r.Next(50) == 0 ? 5:i; // its ok for some projects to have the same number + p.ProjectNumber = r.Next(50) == 0 ? 5:i; // it's ok for some projects to have the same number p.ExtractionDirectory = extractionDir; p.SaveToDatabase(); Projects.Add(1, p); @@ -276,4 +275,4 @@ protected override string[] GetHeaders() throw new NotSupportedException(); } } -} \ No newline at end of file +} diff --git a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs index 8460e106dd..cacc8782ae 100644 --- a/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs +++ b/Rdmp.Core/CommandLine/DatabaseCreation/PlatformDatabaseCreationOptions.cs @@ -50,10 +50,10 @@ public class PlatformDatabaseCreationOptions [Option("NumberOfRowsPerDataset", Default = ExampleDatasetsCreation.NumberOfRowsPerDataset, HelpText = "When ExampleDatasets is set this is the number of rows to create in each dataset")] public int NumberOfRowsPerDataset { get; set; } = ExampleDatasetsCreation.NumberOfRowsPerDataset; - [Option("Nightmare", Default = false, HelpText = "Create a 100,000+ objects in the Catalogue database")] + [Option("Nightmare", Default = false, HelpText = "Create 100,000+ objects in the Catalogue database")] public bool Nightmare { get; set; } - [Option("NightmareFactor", Default = 1, HelpText = "Set to 2 (or more) to double the volume of Nightmare data generated")] + [Option("NightmareFactor", Default = 1, HelpText = "Set to 2 (or more) to multiply the volume of Nightmare data generated")] public int NightmareFactor { get; set; } = 1; [Option(Required = false, Default = false, HelpText = "Set to true to validate the SSL certificate of the server you are installing into")] @@ -100,4 +100,4 @@ public SqlConnectionStringBuilder GetBuilder(string databaseName) return builder; } } -} \ No newline at end of file +} From 56790371c4e7cc5c8cdbc03c8af814c6f6fd026c Mon Sep 17 00:00:00 2001 From: Thomas Nind Date: Fri, 22 Jul 2022 11:28:02 +0100 Subject: [PATCH 10/20] always build nuget packages even when not tagged --- .github/workflows/build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c75ff2239..7bad454b32 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,14 +152,16 @@ jobs: rm dist/rdmp-cli-linux-x64.tar (cd PublishWinForms ; echo 7z a -mx=9 ../dist/rdmp-client.zip . | cmd) - - name: Nuget packages - if: contains(github.ref, 'refs/tags/v') + - name: Build Nuget packages run: | nuget pack Plugins/Plugin/Plugin.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} nuget pack Plugins/Plugin.UI/Plugin.UI.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} nuget pack Plugins/Plugin.Test/Plugin.Test.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} nuget pack Application/ResearchDataManagementPlatform/RDMP.nuspec -Properties Configuration=Release -Version ${{ steps.get_version.outputs.VERSION }} + - name: Push Nuget packages + if: contains(github.ref, 'refs/tags/v') + run: | nuget push HIC.RDMP.Plugin.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} nuget push HIC.RDMP.Plugin.UI.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} nuget push HIC.RDMP.Plugin.Test.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} From d26f90b6472f25dcbc552276e7eaadeca6723107 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 22 Jul 2022 12:48:46 +0000 Subject: [PATCH 11/20] Find RDMP version even on non-tagged builds --- .github/workflows/build.yml | 25 ++++++++----------- .../RDMP.nuspec | 8 +++--- Plugins/Plugin.Test/Plugin.Test.nuspec | 4 --- Plugins/Plugin.UI/Plugin.UI.nuspec | 4 --- 4 files changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7bad454b32..d4ac98b891 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,9 @@ jobs: runs-on: windows-latest steps: + - name: Determine RDMP build version + id: version + run: perl -ne "print \"::set-output name=rdmpversion::$1\" if /AssemblyVersion\(\"([0-9.]+)\"\)/;" SharedAssemblyInfo.cs - name: Stub Node dependencies shell: bash run: touch package-lock.json @@ -126,14 +129,8 @@ jobs: echo "dir /s/b *.cs *.xml > srcbitsa.txt" | cmd perl -pe '$_=reverse' < srcbitsa.txt | sort -t'\' -k1,1 -u | perl -pe '$_=reverse' > srcbits.txt echo 7z a -mx=9 Tools/BundleUpSourceIntoZip/output/SourceCodeForSelfAwareness.zip @srcbits.txt | cmd - - name: Get the version - id: get_version - # Get the tag name e.g. v5.0.0 but skip the 'v' - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\/v/} - shell: bash - name: Sign - if: contains(github.ref, 'refs/tags/v') shell: bash run: | signtool=(/c/Program\ Files\ \(x86\)/Windows\ Kits/10/bin/*/x64/signtool.exe) @@ -142,7 +139,7 @@ jobs: echo ${{ secrets.DIGICERT_PFX }} | base64 --decode > GitHubActionsWorkflow.pfx echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} PublishWindows/*.exe PublishLinux/rdmp PublishWinForms/*.exe PublishWinForms/Hunspell*.dll' | cmd mkdir -p dist - cmd /c wix\\build.cmd ${{ steps.get_version.outputs.VERSION }} + cmd /c wix\\build.cmd ${{ steps.version.outputs.rdmpversion }} echo '"'$signtool'"' 'Sign /f GitHubActionsWorkflow.pfx /fd sha256 /tr http://timestamp.digicert.com /td sha256 /p ${{ secrets.DIGICERT_PASSWORD }} dist/rdmp.msi' | cmd (cd PublishWindows ; echo 7z a -mx=9 ../dist/rdmp-cli-win-x64.zip . | cmd) (cd PublishLinux ; echo 7z a -mx=9 ../dist/rdmp-cli-linux-x64.zip . | cmd) @@ -154,17 +151,17 @@ jobs: - name: Build Nuget packages run: | - nuget pack Plugins/Plugin/Plugin.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} - nuget pack Plugins/Plugin.UI/Plugin.UI.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} - nuget pack Plugins/Plugin.Test/Plugin.Test.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.get_version.outputs.VERSION }} - nuget pack Application/ResearchDataManagementPlatform/RDMP.nuspec -Properties Configuration=Release -Version ${{ steps.get_version.outputs.VERSION }} + nuget pack Plugins/Plugin/Plugin.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.version.outputs.rdmpversion }} + nuget pack Plugins/Plugin.UI/Plugin.UI.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.version.outputs.rdmpversion }} + nuget pack Plugins/Plugin.Test/Plugin.Test.nuspec -Properties Configuration=Release -IncludeReferencedProjects -Symbols -Version ${{ steps.version.outputs.rdmpversion }} + nuget pack Application/ResearchDataManagementPlatform/RDMP.nuspec -Properties Configuration=Release -Version ${{ steps.version.outputs.rdmpversion }} - name: Push Nuget packages if: contains(github.ref, 'refs/tags/v') run: | - nuget push HIC.RDMP.Plugin.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} - nuget push HIC.RDMP.Plugin.UI.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} - nuget push HIC.RDMP.Plugin.Test.${{ steps.get_version.outputs.VERSION }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} + nuget push HIC.RDMP.Plugin.${{ steps.version.outputs.rdmpversion }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} + nuget push HIC.RDMP.Plugin.UI.${{ steps.version.outputs.rdmpversion }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} + nuget push HIC.RDMP.Plugin.Test.${{ steps.version.outputs.rdmpversion }}.nupkg -skipDuplicate -Source https://api.nuget.org/v3/index.json -ApiKey ${{ secrets.NUGET_KEY }} - name: Archive production artifacts uses: actions/upload-artifact@v3 diff --git a/Application/ResearchDataManagementPlatform/RDMP.nuspec b/Application/ResearchDataManagementPlatform/RDMP.nuspec index a1abd05963..2c0a9ed8eb 100644 --- a/Application/ResearchDataManagementPlatform/RDMP.nuspec +++ b/Application/ResearchDataManagementPlatform/RDMP.nuspec @@ -7,9 +7,9 @@ Research Data Management Platform - - - - + + + + \ No newline at end of file diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 925d96bd7b..3967f4861b 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -44,16 +44,12 @@ - - - - diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index 7e11b486ac..6584366397 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -48,16 +48,12 @@ - - - - From aadb050876197a6c13ea8cd911a6b3db7bc4eee0 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 22 Jul 2022 12:54:12 +0000 Subject: [PATCH 12/20] - Bypass Powershell stupidity --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d4ac98b891..41fd94e8c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,7 @@ jobs: steps: - name: Determine RDMP build version id: version + shell: cmd run: perl -ne "print \"::set-output name=rdmpversion::$1\" if /AssemblyVersion\(\"([0-9.]+)\"\)/;" SharedAssemblyInfo.cs - name: Stub Node dependencies shell: bash From 5ab97741fc659429e0c3e0222217dc329cc832da Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 22 Jul 2022 16:45:08 +0000 Subject: [PATCH 13/20] Parse SharedAssemblyInfo after code checkout --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41fd94e8c5..4d09a56dbd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,10 +14,6 @@ jobs: runs-on: windows-latest steps: - - name: Determine RDMP build version - id: version - shell: cmd - run: perl -ne "print \"::set-output name=rdmpversion::$1\" if /AssemblyVersion\(\"([0-9.]+)\"\)/;" SharedAssemblyInfo.cs - name: Stub Node dependencies shell: bash run: touch package-lock.json @@ -30,6 +26,10 @@ jobs: run: npm install -g lcov-result-merger - name: Checkout code uses: actions/checkout@v3 + - name: Determine RDMP build version + id: version + shell: cmd + run: perl -ne "print \"::set-output name=rdmpversion::$1\" if /AssemblyVersion\(\"([0-9.]+)\"\)/;" SharedAssemblyInfo.cs - uses: actions/cache@v3 with: path: ~/.nuget/packages From ae029f4c961f6ee4845a1e201739cfdee45da265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 00:14:44 +0000 Subject: [PATCH 14/20] Bump YamlDotNet from 11.2.1 to 12.0.0 Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 11.2.1 to 12.0.0. - [Release notes](https://github.com/aaubry/YamlDotNet/releases) - [Commits](https://github.com/aaubry/YamlDotNet/compare/v11.2.1...v12.0.0) --- updated-dependencies: - dependency-name: YamlDotNet dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- Tests.Common/Tests.Common.csproj | 2 +- Tools/rdmp/rdmp.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 0286a39db0..217e9a9758 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -284,7 +284,7 @@ - + diff --git a/Tests.Common/Tests.Common.csproj b/Tests.Common/Tests.Common.csproj index fb2e5e035d..3a0d63e041 100644 --- a/Tests.Common/Tests.Common.csproj +++ b/Tests.Common/Tests.Common.csproj @@ -30,7 +30,7 @@ - + diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index d13fd7c568..4dc63e7be9 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -43,7 +43,7 @@ - + From c9c42288e87075db4be4b6128f65c8038e3e0a7c Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Sun, 24 Jul 2022 19:48:23 -0500 Subject: [PATCH 15/20] Bump YamlDotNet from 11.2.1 to 12.0.0 --- Documentation/CodeTutorials/Packages.md | 2 +- Plugins/Plugin.Test/Plugin.Test.nuspec | 2 +- Plugins/Plugin.UI/Plugin.UI.nuspec | 2 +- Plugins/Plugin/Plugin.nuspec | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/CodeTutorials/Packages.md b/Documentation/CodeTutorials/Packages.md index db48d4387c..8e85e68227 100644 --- a/Documentation/CodeTutorials/Packages.md +++ b/Documentation/CodeTutorials/Packages.md @@ -29,7 +29,7 @@ | Microsoft.NET.Test.Sdk | [GitHub](https://github.com/microsoft/vstest/) | [17.2.0](https://www.nuget.org/packages/Microsoft.NET.Test.Sdk/17.2.0) | [MIT](https://opensource.org/licenses/MIT) | Required for running tests| | | NUnit3TestAdapter | [GitHub](https://github.com/nunit/nunit3-vs-adapter)| [3.13.3](https://www.nuget.org/packages/NUnit3TestAdapter/3.13.3) | [MIT](https://opensource.org/licenses/MIT) | Run unit tests from within Visual Studio | | [Newtonsoft.Json](https://www.newtonsoft.com/json) | [GitHub](https://github.com/JamesNK/Newtonsoft.Json) | [13.0.1](https://www.nuget.org/packages/Newtonsoft.Json/13.0.1) | [MIT](https://opensource.org/licenses/MIT) | Serialization of objects for sharing/transmission | -| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [11.2.1](https://www.nuget.org/packages/YamlDotNet/11.2.1) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| +| YamlDotNet | [GitHub](https://github.com/aaubry/YamlDotNet) | [12.0.0](https://www.nuget.org/packages/YamlDotNet/12.0.0) | [MIT](https://opensource.org/licenses/MIT) |Loading configuration files| | [SecurityCodeScan.VS2019](https://security-code-scan.github.io/) | [GitHub](https://github.com/security-code-scan/security-code-scan) | [5.6.3](https://www.nuget.org/packages/SecurityCodeScan.VS2019/5.6.3) | [GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.html)| Performs static build time analysis for vulnerabilities in the codebase (e.g. Sql injection)| | | System.Runtime.Loader | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.Runtime.Loader/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | Allows loading assemblies in dot net core| | | System.Diagnostics.Debug | [GitHub](https://github.com/dotnet/corefx) | [4.3.0](https://www.nuget.org/packages/System.Diagnostics.Debug/4.3.0) |[MIT](https://opensource.org/licenses/MIT) | Interact with Processes / Debug / Console | | diff --git a/Plugins/Plugin.Test/Plugin.Test.nuspec b/Plugins/Plugin.Test/Plugin.Test.nuspec index 925d96bd7b..419918b655 100644 --- a/Plugins/Plugin.Test/Plugin.Test.nuspec +++ b/Plugins/Plugin.Test/Plugin.Test.nuspec @@ -22,7 +22,7 @@ - + diff --git a/Plugins/Plugin.UI/Plugin.UI.nuspec b/Plugins/Plugin.UI/Plugin.UI.nuspec index 7e11b486ac..b228855b2f 100644 --- a/Plugins/Plugin.UI/Plugin.UI.nuspec +++ b/Plugins/Plugin.UI/Plugin.UI.nuspec @@ -43,7 +43,7 @@ - + diff --git a/Plugins/Plugin/Plugin.nuspec b/Plugins/Plugin/Plugin.nuspec index 260485b48e..3b4352282b 100644 --- a/Plugins/Plugin/Plugin.nuspec +++ b/Plugins/Plugin/Plugin.nuspec @@ -36,7 +36,7 @@ - + From 5869280cb9ab797a28e4f9e9fe7354df44426e0f Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 25 Jul 2022 07:30:25 -0500 Subject: [PATCH 16/20] Revert publishsinglefile csproj for now pending VS bug fix to restore debugging --- .../ResearchDataManagementPlatform.csproj | 2 +- Tools/rdmp/rdmp.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj index 1ec9ee14ae..4269a465ac 100644 --- a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj +++ b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj @@ -11,7 +11,7 @@ 1701;1702;CS1591;NU1701 embedded true - true + false>truefalse> diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index d13fd7c568..80d4ce3660 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -12,7 +12,7 @@ Rdmp.Core embedded true - true + false>truefalse> true From d474aa10b58c1f1756298ba2db7c50d434b85358 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 25 Jul 2022 08:04:46 -0500 Subject: [PATCH 17/20] Fix XML slip --- .../ResearchDataManagementPlatform.csproj | 2 +- Tools/rdmp/rdmp.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj index 4269a465ac..ad41b67733 100644 --- a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj +++ b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj @@ -11,7 +11,7 @@ 1701;1702;CS1591;NU1701 embedded true - false>truefalse> + false diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index 80d4ce3660..c4235c0d57 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -12,7 +12,7 @@ Rdmp.Core embedded true - false>truefalse> + false true From 040620055020432e1842b98fe3094b2706b23243 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 25 Jul 2022 10:17:09 -0500 Subject: [PATCH 18/20] - Ensure RDMP core types are loaded even if we aren't in a DLL - Revert PublishSingleFile in csproj for now since it breaks VS debugging: https://github.com/dotnet/runtime/issues/45382 - Clearer null checks so ReSharper knows it's safe - Fix typos - Use nameof for argument name string --- .../Curation/Data/SafeDirectoryCatalog.cs | 45 ++++++++++--------- Rdmp.Core/Repositories/MEF.cs | 2 +- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/Rdmp.Core/Curation/Data/SafeDirectoryCatalog.cs b/Rdmp.Core/Curation/Data/SafeDirectoryCatalog.cs index 9e36848f66..fbeb1e8ecd 100644 --- a/Rdmp.Core/Curation/Data/SafeDirectoryCatalog.cs +++ b/Rdmp.Core/Curation/Data/SafeDirectoryCatalog.cs @@ -29,18 +29,17 @@ public class SafeDirectoryCatalog /// /// These assemblies do not load correctly and should be ignored (they produce warnings on Startup) /// - public static string[] Ignore = new string[] - { - "mscorelib.dll", + public static string[] Ignore = { + "mscorlib.dll", "Hunspellx64.dll", - "Hunspellx86.dll", + "Hunspellx86.dll" }; /// - /// Assemblies succesfully loaded + /// Assemblies successfully loaded /// - public ConcurrentDictionary GoodAssemblies = new (); - public ConcurrentDictionary TypesByAssembly = new (); + public readonly ConcurrentDictionary GoodAssemblies = new (); + public readonly ConcurrentDictionary TypesByAssembly = new (); private object oTypesLock = new object(); public HashSet Types = new HashSet(); @@ -77,12 +76,17 @@ public SafeDirectoryCatalog(ICheckNotifier listener, params string[] directories { BadAssembliesDictionary = new Dictionary(); + TypesByAssembly.TryAdd(typeof(SafeDirectoryCatalog).Assembly, + typeof(SafeDirectoryCatalog).Assembly.GetTypes()); + foreach(var t in typeof(SafeDirectoryCatalog).Assembly.GetTypes()) + AddType(t); + var files = new HashSet(); - foreach (string directory in directories) + foreach (var directory in directories) { - if (!Directory.Exists(directory)) - Directory.CreateDirectory(directory);//empty directory + if (directory != null && !Directory.Exists(directory)) + Directory.CreateDirectory(directory); //empty directory foreach(var f in Directory.EnumerateFiles(directory, "*.dll", SearchOption.AllDirectories)) { @@ -98,7 +102,7 @@ public SafeDirectoryCatalog(ICheckNotifier listener, params string[] directories var existingOneVersion = FileVersionInfo.GetVersionInfo(existing.FullName); var newOneVersion = FileVersionInfo.GetVersionInfo(newOne.FullName); - FileInfo winner = null; + FileInfo winner; // if we already have a copy of this exact dll we don't care about loading it if(FileVersionsAreEqual(newOneVersion, existingOneVersion)) @@ -215,20 +219,17 @@ private static bool FileVersionGreaterThan(FileVersionInfo newOneVersion, FileVe private void AddBadAssembly(FileInfo f, Exception ex,ICheckNotifier listener) { - if (!BadAssembliesDictionary.ContainsKey(f.FullName)) //couldn't load anything out of it - { - BadAssembliesDictionary.Add(f.FullName, ex); - - listener?.OnCheckPerformed(new CheckEventArgs(ErrorCodes.CouldNotLoadDll, null,ex,f.FullName)); - } + if (BadAssembliesDictionary.ContainsKey(f.FullName)) return; // Only report each failure once + BadAssembliesDictionary.Add(f.FullName, ex); + listener?.OnCheckPerformed(new CheckEventArgs(ErrorCodes.CouldNotLoadDll, null,ex,f.FullName)); } private void AddTypes(FileInfo f, Assembly ass, Type[] types, ICheckNotifier listener) { TypesByAssembly.TryAdd(ass,types.Where(t=>t != null).ToArray()); - foreach(Type t in types.Where(t=>t != null)) - if(!TypesByName.ContainsKey(t.FullName)) + foreach(var t in types) + if(t.FullName != null && !TypesByName.ContainsKey(t.FullName)) AddType(t.FullName,t); GoodAssemblies.TryAdd(f.FullName, ass); @@ -244,9 +245,9 @@ internal void AddType(Type type) internal void AddType(string typeNameOrAlias, Type type) { - //only add it if it is novel - if(!TypesByName.ContainsKey(typeNameOrAlias)) - TypesByName.TryAdd(typeNameOrAlias,type); + //only add it if it is novel + if (!TypesByName.ContainsKey(typeNameOrAlias)) + TypesByName.TryAdd(typeNameOrAlias, type); lock (oTypesLock) { diff --git a/Rdmp.Core/Repositories/MEF.cs b/Rdmp.Core/Repositories/MEF.cs index 5fd250e0ae..43fd005756 100644 --- a/Rdmp.Core/Repositories/MEF.cs +++ b/Rdmp.Core/Repositories/MEF.cs @@ -70,7 +70,7 @@ public MEF() public Type GetType(string type) { if(string.IsNullOrWhiteSpace(type)) - throw new ArgumentNullException("type"); + throw new ArgumentNullException(nameof(type)); if(TypeNotKnown.Contains(type)) return null; From 6f066a2a0adefbe885340cd1ba9e9a885f3bc567 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 25 Jul 2022 16:11:23 -0500 Subject: [PATCH 19/20] Skip publishsinglefile due to VS bug for now --- Tools/rdmp/rdmp.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index 4dc63e7be9..0ca05bf6d6 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -12,7 +12,7 @@ Rdmp.Core embedded true - true + false true From 46d74c33096f38f230700d0551e720d02415382a Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Mon, 25 Jul 2022 17:24:19 -0500 Subject: [PATCH 20/20] Prepare for release as v7.0.16 --- CHANGELOG.md | 5 +++++ SharedAssemblyInfo.cs | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 836d8ca4bd..3e59afec0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ... +## [7.0.16] - 2022-07-25 + +- Bugfix release due to build issues in releasing 7.0.15 +- Bump YamlDotNet from 11.2.1 to 12.0.0 + ## [7.0.15] - 2022-07-22 ### Added diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 92ac019d64..f62cd7373f 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -6,6 +6,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("7.0.15")] -[assembly: AssemblyFileVersion("7.0.15")] -[assembly: AssemblyInformationalVersion("7.0.15")] +[assembly: AssemblyVersion("7.0.16")] +[assembly: AssemblyFileVersion("7.0.16")] +[assembly: AssemblyInformationalVersion("7.0.16")]