diff --git a/.gitignore b/.gitignore index 8a30d25..5ae91ed 100644 --- a/.gitignore +++ b/.gitignore @@ -396,3 +396,5 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml + +switcher.json diff --git a/DEH-CSV.Tests/CsvReaderTestFixture.cs b/DEH-CSV.Tests/CsvReaderTestFixture.cs new file mode 100644 index 0000000..918001e --- /dev/null +++ b/DEH-CSV.Tests/CsvReaderTestFixture.cs @@ -0,0 +1,159 @@ +// ------------------------------------------------------------------------------------------------- +// +// +// Copyright 2023-2024 RHEA System S.A. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// ------------------------------------------------------------------------------------------------- + +namespace RHEAGROUP.DEHCSV.Tests +{ + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + + using CDP4Common.CommonData; + using CDP4Common.EngineeringModelData; + using CDP4Common.SiteDirectoryData; + + using CDP4Dal; + using CDP4Dal.DAL; + + using CDP4ServicesDal; + + using Microsoft.Extensions.Logging; + + using NUnit.Framework; + + using RHEAGROUP.DEHCSV.Mapping; + + using File = System.IO.File; + + [TestFixture] + public class CsvReaderTestFixture + { + private readonly Uri uri = new("https://cdp4services-public.cdp4.org"); + private Credentials credentials; + private CdpServicesDal dal; + private CDPMessageBus messageBus; + private Session session; + private CsvReader csvReader; + private JsonSerializerOptions options; + + [SetUp] + public void Setup() + { + this.credentials = new Credentials("admin", "pass", this.uri); + this.dal = new CdpServicesDal(); + this.messageBus = new CDPMessageBus(); + this.session = new Session(this.dal, this.credentials, this.messageBus); + var loggerFactory = LoggerFactory.Create(x => x.AddConsole()); + + this.csvReader = new CsvReader(loggerFactory.CreateLogger()); + + this.options = new JsonSerializerOptions() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + Converters = { new JsonStringEnumConverter() } + }; + } + + [TearDown] + public void Teardown() + { + this.messageBus.Dispose(); + } + + [Test] + public async Task VerifyCsvReaderImplementation() + { + CultureInfo.CurrentCulture = CultureInfo.InvariantCulture; + + var csvPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Data", "import-data.csv"); + var mappingFunctionPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Data", "import-mapping.json"); + + var typeMaps = JsonSerializer.Deserialize>(await File.ReadAllTextAsync(mappingFunctionPath), this.options); + + var csvStream = File.OpenRead(csvPath); + await this.session.Open(); + var loftModel = this.session.RetrieveSiteDirectory().Model.Find(x => x.Name == "LOFT")!; + var iterationSetup = loftModel.IterationSetup.Single(x => x.FrozenOn == null); + + var iteration = new Iteration() + { + Iid = iterationSetup.IterationIid, + IterationSetup = iterationSetup + }; + + var engineeringModel = new EngineeringModel() + { + Iid = loftModel.EngineeringModelIid, + EngineeringModelSetup = loftModel + }; + + engineeringModel.Iteration.Add(iteration); + + var domain = loftModel.Participant.Single(x => x.Person == this.session.ActivePerson).SelectedDomain; + await this.session.Read(iteration, domain); + var mappedThings = (await this.csvReader.Read(csvStream, typeMaps.ToList(), this.session)).ToList(); + + Assert.Multiple(() => + { + Assert.That(mappedThings, Is.Not.Empty); + Assert.That(mappedThings, Has.Count.EqualTo(315)); + Assert.That(mappedThings.OfType().ToImmutableList(), Has.Count.EqualTo(35)); + Assert.That(mappedThings.OfType().ToImmutableList(), Has.Count.EqualTo(140)); + Assert.That(mappedThings.OfType().ToImmutableList(), Has.Count.EqualTo(140)); + }); + + var count = 0; + + foreach (var elementDefinition in mappedThings.OfType()) + { + Assert.Multiple(() => + { + Assert.That(elementDefinition.ShortName, Is.EqualTo($"shortName{count:0000}")); + Assert.That(elementDefinition.Name, Is.EqualTo($"Name {count:0000}")); + Assert.That(elementDefinition.Category[0].Name, Is.EqualTo("Subsystem")); + Assert.That(elementDefinition.Owner.Name, Is.EqualTo("System Engineering")); + Assert.That(elementDefinition.Parameter, Has.Count.EqualTo(4)); + Assert.That(elementDefinition.Parameter[0].ParameterType.Name, Is.EqualTo("area")); + Assert.That(elementDefinition.Parameter[1].ParameterType.Name, Is.EqualTo("mass")); + Assert.That(elementDefinition.Parameter[2].ParameterType.Name, Is.EqualTo("dry mass")); + Assert.That(elementDefinition.Parameter[3].ParameterType.Name, Is.EqualTo("radius")); + Assert.That(elementDefinition.Parameter.Select(x => x.Owner).Distinct(), Is.EquivalentTo(new List { elementDefinition.Owner })); + }); + + foreach (var parameter in elementDefinition.Parameter) + { + Assert.That(parameter.ValueSet, Has.Count.EqualTo(1)); + } + + Assert.That(int.Parse(elementDefinition.Parameter[0].ValueSet[0].Manual[0]), Is.EqualTo(count)); + Assert.That(int.Parse(elementDefinition.Parameter[1].ValueSet[0].Manual[0]), Is.EqualTo(-count)); + Assert.That(int.Parse(elementDefinition.Parameter[2].ValueSet[0].Manual[0]), Is.EqualTo(count + 10)); + Assert.That(int.Parse(elementDefinition.Parameter[3].ValueSet[0].Manual[0]), Is.EqualTo(count + 100)); + + count++; + } + } + } +} diff --git a/DEH-CSV.Tests/CsvWriterTestFixture.cs b/DEH-CSV.Tests/CsvWriterTestFixture.cs index eb97322..02eecde 100644 --- a/DEH-CSV.Tests/CsvWriterTestFixture.cs +++ b/DEH-CSV.Tests/CsvWriterTestFixture.cs @@ -54,6 +54,7 @@ public class CsvWriterTestFixture private Uri uri; private string mappingPath; + private CDPMessageBus messageBus; [SetUp] public void SetUp() @@ -66,10 +67,16 @@ public void SetUp() this.mappingProvider = new MappingProvider(this.loggerFactory); this.iterationReader = new IterationReader(this.loggerFactory); - + this.messageBus = new CDPMessageBus(); this.csvWriter = new CsvWriter(this.loggerFactory); } + [TearDown] + public void Teardown() + { + this.messageBus.Dispose(); + } + [Test] public async Task Verify_that_demosat_model_can_be_written_to_CSV_file() { @@ -80,8 +87,8 @@ public async Task Verify_that_demosat_model_can_be_written_to_CSV_file() this.uri = new Uri(path); var credentials = new Credentials("admin", "pass", uri); - - var session = new Session(jsonFileDal, credentials); + + var session = new Session(jsonFileDal, credentials, this.messageBus); await session.Open(false); @@ -106,7 +113,7 @@ public async Task Verify_that_when_ValuePrefix_is_set_CSV_File_is_Written_with_p var credentials = new Credentials("admin", "pass", uri); - var session = new Session(jsonFileDal, credentials); + var session = new Session(jsonFileDal, credentials, this.messageBus); await session.Open(false); diff --git a/DEH-CSV.Tests/CustomProperties/ThingTimeStampedCSVWriterTestFixture.cs b/DEH-CSV.Tests/CustomProperties/ThingTimeStampedCSVWriterTestFixture.cs index 5680f24..93ce9b6 100644 --- a/DEH-CSV.Tests/CustomProperties/ThingTimeStampedCSVWriterTestFixture.cs +++ b/DEH-CSV.Tests/CustomProperties/ThingTimeStampedCSVWriterTestFixture.cs @@ -54,6 +54,8 @@ public class ThingTimeStampedCSVWriterTestFixture private string mappingPath; + private CDPMessageBus messageBus; + [SetUp] public void SetUp() { @@ -64,6 +66,14 @@ public void SetUp() this.iterationReader = new IterationReader(); this.thingTimeStampedCsvWriter = new ThingTimeStampedCSVWriter(); + + this.messageBus = new CDPMessageBus(); + } + + [TearDown] + public void Teardown() + { + this.messageBus.Dispose(); } [Test] @@ -77,7 +87,7 @@ public async Task Verify_that_demosat_model_can_be_written_to_CSV_file() var credentials = new Credentials("admin", "pass", uri); - var session = new Session(jsonFileDal, credentials); + var session = new Session(jsonFileDal, credentials, this.messageBus); await session.Open(false); diff --git a/DEH-CSV.Tests/DEH-CSV.Tests.csproj b/DEH-CSV.Tests/DEH-CSV.Tests.csproj index 11cc1d0..e18baee 100644 --- a/DEH-CSV.Tests/DEH-CSV.Tests.csproj +++ b/DEH-CSV.Tests/DEH-CSV.Tests.csproj @@ -59,6 +59,12 @@ Always + + Always + + + Always + Always diff --git a/DEH-CSV.Tests/Data/import-data.csv b/DEH-CSV.Tests/Data/import-data.csv new file mode 100644 index 0000000..491a5fc --- /dev/null +++ b/DEH-CSV.Tests/Data/import-data.csv @@ -0,0 +1,36 @@ +Category;Short name;Name;area;mass;dry mass;radius;Ownership +Subsystem;shortName0000;Name 0000;0;0;10;100;System Engineering +Subsystem;shortName0001;Name 0001;1;-1;11;101;System Engineering +Subsystem;shortName0002;Name 0002;2;-2;12;102;System Engineering +Subsystem;shortName0003;Name 0003;3;-3;13;103;System Engineering +Subsystem;shortName0004;Name 0004;4;-4;14;104;System Engineering +Subsystem;shortName0005;Name 0005;5;-5;15;105;System Engineering +Subsystem;shortName0006;Name 0006;6;-6;16;106;System Engineering +Subsystem;shortName0007;Name 0007;7;-7;17;107;System Engineering +Subsystem;shortName0008;Name 0008;8;-8;18;108;System Engineering +Subsystem;shortName0009;Name 0009;9;-9;19;109;System Engineering +Subsystem;shortName0010;Name 0010;10;-10;20;110;System Engineering +Subsystem;shortName0011;Name 0011;11;-11;21;111;System Engineering +Subsystem;shortName0012;Name 0012;12;-12;22;112;System Engineering +Subsystem;shortName0013;Name 0013;13;-13;23;113;System Engineering +Subsystem;shortName0014;Name 0014;14;-14;24;114;System Engineering +Subsystem;shortName0015;Name 0015;15;-15;25;115;System Engineering +Subsystem;shortName0016;Name 0016;16;-16;26;116;System Engineering +Subsystem;shortName0017;Name 0017;17;-17;27;117;System Engineering +Subsystem;shortName0018;Name 0018;18;-18;28;118;System Engineering +Subsystem;shortName0019;Name 0019;19;-19;29;119;System Engineering +Subsystem;shortName0020;Name 0020;20;-20;30;120;System Engineering +Subsystem;shortName0021;Name 0021;21;-21;31;121;System Engineering +Subsystem;shortName0022;Name 0022;22;-22;32;122;System Engineering +Subsystem;shortName0023;Name 0023;23;-23;33;123;System Engineering +Subsystem;shortName0024;Name 0024;24;-24;34;124;System Engineering +Subsystem;shortName0025;Name 0025;25;-25;35;125;System Engineering +Subsystem;shortName0026;Name 0026;26;-26;36;126;System Engineering +Subsystem;shortName0027;Name 0027;27;-27;37;127;System Engineering +Subsystem;shortName0028;Name 0028;28;-28;38;128;System Engineering +Subsystem;shortName0029;Name 0029;29;-29;39;129;System Engineering +Subsystem;shortName0030;Name 0030;30;-30;40;130;System Engineering +Subsystem;shortName0031;Name 0031;31;-31;41;131;System Engineering +Subsystem;shortName0032;Name 0032;32;-32;42;132;System Engineering +Subsystem;shortName0033;Name 0033;33;-33;43;133;System Engineering +Subsystem;shortName0034;Name 0034;34;-34;44;134;System Engineering diff --git a/DEH-CSV.Tests/Data/import-mapping.json b/DEH-CSV.Tests/Data/import-mapping.json new file mode 100644 index 0000000..740d176 --- /dev/null +++ b/DEH-CSV.Tests/Data/import-mapping.json @@ -0,0 +1,162 @@ +[ + { + "classKind": "ElementDefinition", + "properties": [ + { + "source": "Short name", + "classKind": "ElementDefinition", + "search": "ShortName", + "isIdentifierProperty": true, + "target": "ShortName" + }, + { + "source": "Category", + "searchClassKind": "Category", + "search": "Name", + "target": "Category[0..*]" + }, + { + "source": "Name", + "target": "Name" + }, + { + "source": "Ownership", + "searchClassKind": "DomainOfExpertise", + "search": "Name", + "target": "Owner" + } + ] + }, + { + "classKind": "Parameter", + "properties": [ + { + "source": "Short name", + "classKind": "ElementDefinition", + "search": "ShortName", + "isIdentifierProperty": true + }, + { + "source": "area", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]", + "target": "ParameterType", + "searchClassKind": "DerivedQuantityKind" + }, + { + "source": "mass", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]", + "target": "ParameterType", + "searchClassKind": "SimpleQuantityKind" + }, + { + "source": "dry mass", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]", + "target": "ParameterType", + "searchClassKind": "SpecializedQuantityKind" + }, + { + "source": "radius", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]", + "target": "ParameterType", + "searchClassKind": "SpecializedQuantityKind" + }, + { + "source": "Ownership", + "searchClassKind": "DomainOfExpertise", + "search": "Name", + "target": "Owner" + } + ] + }, + { + "classKind": "ParameterValueSet", + "properties": [ + { + "source": "Short name", + "classKind": "ElementDefinition", + "search": "ShortName", + "isIdentifierProperty": true + }, + { + "source": "area", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]" + }, + { + "source": "area", + "classKind": "ParameterValueSet", + "isIdentifierProperty": true, + "path": "Parameter[0..*].ValueSet[0..*]", + "target": "Manual", + "firstOrDefault": true + }, + { + "source": "mass", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]" + }, + { + "source": "mass", + "classKind": "ParameterValueSet", + "isIdentifierProperty": true, + "path": "Parameter[0..*].ValueSet[0..*]", + "target": "Manual", + "firstOrDefault": true + }, + { + "source": "dry mass", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]" + }, + { + "source": "dry mass", + "classKind": "ParameterValueSet", + "isIdentifierProperty": true, + "path": "Parameter[0..*].ValueSet[0..*]", + "target": "Manual", + "firstOrDefault": true + }, + { + "source": "radius", + "classKind": "Parameter", + "search": "ParameterType.Name", + "isIdentifierProperty": true, + "searchBasedOnHeader": true, + "path": "Parameter[0..*]" + }, + { + "source": "radius", + "classKind": "ParameterValueSet", + "isIdentifierProperty": true, + "path": "Parameter[0..*].ValueSet[0..*]", + "target": "Manual", + "firstOrDefault": true + } + ] + } +] \ No newline at end of file diff --git a/DEH-CSV.Tests/Services/IterationReaderTestFixture.cs b/DEH-CSV.Tests/Services/IterationReaderTestFixture.cs index 9112966..e4245fb 100644 --- a/DEH-CSV.Tests/Services/IterationReaderTestFixture.cs +++ b/DEH-CSV.Tests/Services/IterationReaderTestFixture.cs @@ -42,6 +42,7 @@ public class IterationReaderTestFixture private ILoggerFactory loggerFactory; private IterationReader iterationReader; + private CDPMessageBus messageBus; [SetUp] public void SetUp() @@ -50,6 +51,14 @@ public void SetUp() builder.AddConsole().SetMinimumLevel(LogLevel.Trace)); this.iterationReader = new IterationReader(this.loggerFactory); + + this.messageBus = new CDPMessageBus(); + } + + [TearDown] + public void Teardown() + { + this.messageBus.Dispose(); } [Test] @@ -63,7 +72,7 @@ public async Task Verify_that_iteration_can_be_read_from_data_source_demo_space( var credentials = new Credentials("admin", "pass", uri); - var session = new Session(jsonFileDal, credentials); + var session = new Session(jsonFileDal, credentials, this.messageBus); await session.Open(false); @@ -83,7 +92,7 @@ public async Task Verify_that_iteration_read_throws_expected_exceptions() var credentials = new Credentials("admin", "pass", uri); - var session = new Session(jsonFileDal, credentials); + var session = new Session(jsonFileDal, credentials, this.messageBus); await session.Open(false); diff --git a/DEH-CSV.sln.DotSettings b/DEH-CSV.sln.DotSettings index e9c5705..9662113 100644 --- a/DEH-CSV.sln.DotSettings +++ b/DEH-CSV.sln.DotSettings @@ -3,9 +3,9 @@ Field, Property, Event, Method True ------------------------------------------------------------------------------------------------- - <copyright file="$FILENAME$" company="RHEA System S.A."> + <copyright file="${File.FileName}" company="RHEA System S.A."> - Copyright 2023 RHEA System S.A. + Copyright 2023-${CurrentDate.Year} RHEA System S.A. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ ------------------------------------------------------------------------------------------------- <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> + True True True True diff --git a/DEH-CSV/CsvReader.cs b/DEH-CSV/CsvReader.cs new file mode 100644 index 0000000..0d9a343 --- /dev/null +++ b/DEH-CSV/CsvReader.cs @@ -0,0 +1,535 @@ +// ------------------------------------------------------------------------------------------------- +// +// +// Copyright 2023-2024 RHEA System S.A. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// ------------------------------------------------------------------------------------------------- + +namespace RHEAGROUP.DEHCSV +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + + using CDP4Common.CommonData; + using CDP4Common.Helpers; + using CDP4Common.PropertyAccesor; + + using CDP4Dal; + + using CsvHelper; + using CsvHelper.Configuration; + + using Microsoft.Extensions.Logging; + + using RHEAGROUP.DEHCSV.Helpers; + using RHEAGROUP.DEHCSV.Mapping; + + /// + /// The purpose of the is to read CSV files and transform the content to + /// ECSS-E-TM-10-25 data set based on s + /// + public class CsvReader : ICsvReader + { + /// + /// Gets the injected + /// + private readonly ILogger logger; + + /// + /// Initializes a new instance of + /// + /// The + public CsvReader(ILogger logger) + { + this.logger = logger; + } + + /// + /// Reads the CSV content of the and maps it to s based on the provided collection of + /// s + /// + /// The that contains CSV content + /// The collection of s + /// The that helps to retrieve + /// A that returns a collection of mapped s + public async Task> Read(Stream stream, IReadOnlyCollection typeMaps, ISession session) + { + ValidateReadParameters(stream, typeMaps, session); + + var things = new List(); + + var accessibleThings = session.Assembler.Cache + .Where(x => x.Value.IsValueCreated) + .Select(x => x.Value.Value) + .ToImmutableList(); + + stream.Position = 0; + using var streamReader = new StreamReader(stream); + + using var reader = new CsvHelper.CsvReader(streamReader, new CsvConfiguration(CultureInfo.InvariantCulture) + { + DetectDelimiter = true + }); + + await this.ReadHeader(reader, typeMaps); + + while (await reader.ReadAsync()) + { + foreach (var typeMap in typeMaps) + { + this.MapCsvRow(reader, typeMap, accessibleThings, things); + } + } + + return things; + } + + /// + /// Map a row of the CSV file to + /// + /// The used to read CSV content + /// A to process + /// A of s that comes from the cache + /// The collection of all s that have been mapped for a CSV file + private void MapCsvRow(IReaderRow reader, TypeMap typeMap, IReadOnlyCollection accessibleThings, List allMappedThings) + { + var mappedThings = new List(); + + var path = new PropertyPath(typeMap.Properties); + + var currentPropertyMap = typeMap.Properties.Single(x => x.IsIdentifierProperty && string.IsNullOrEmpty(x.Path)); + + this.ValidateEntryPoint(typeMap, currentPropertyMap); + + var currentClasskind = currentPropertyMap.ClassKind!.Value; + var alreadyReadThings = new List(allMappedThings); + var entryPointValue = QueryValueToUse(reader, currentPropertyMap); + var lastThingsBeforeTargetClassKind = new Dictionary<(Thing Thing, string PropertyName), List>(); + + this.logger.LogDebug("Processing Entry point with ClassKind {0}", currentClasskind); + + var previousThings = QueryMatchingThings(currentClasskind, entryPointValue, currentPropertyMap, alreadyReadThings, accessibleThings); + + if (currentPropertyMap.ClassKind == typeMap.ClassKind) + { + if (previousThings.Count != 0) + { + foreach (var previousThing in previousThings) + { + UpdateThingValues(reader, previousThing, typeMap, currentPropertyMap, alreadyReadThings, accessibleThings); + } + + mappedThings.AddRange(previousThings); + } + else + { + var newThing = TypeInitializer.Initialize(currentClasskind); + UpdateThingValues(reader, newThing, typeMap, currentPropertyMap, alreadyReadThings, accessibleThings); + mappedThings.Add(newThing); + } + } + else + { + if (previousThings.Count == 0) + { + this.logger.LogError("The provided CSV references Thing(s) that are not part of the Database: Source : {0}", path.PropertyMap.Source); + throw new InvalidDataException($"The provided CSV references Thing(s) that are not part of the Database: Source : {path.PropertyMap.Source}"); + } + + foreach (var child in path.Children) + { + this.ProcessPath(child, previousThings, reader, alreadyReadThings, accessibleThings, lastThingsBeforeTargetClassKind, typeMap, mappedThings); + } + } + + foreach (var kvp in lastThingsBeforeTargetClassKind) + { + // Handle case if the value to set is a not a collection + if (kvp.Value.Count == 1) + { + kvp.Key.Thing.SetValue(kvp.Key.PropertyName, kvp.Value[0]); + } + else + { + kvp.Key.Thing.SetValue(kvp.Key.PropertyName, kvp.Value); + } + } + + allMappedThings.AddRange(mappedThings.Distinct().Where(x => !allMappedThings.Contains(x))); + } + + /// + /// Process a to map CSV content to s + /// + /// The to process + /// + /// A collection of that are part of the previous + /// + /// + /// The to be able to read CSV content + /// A collection of already read s + /// A of s that comes from the cache + /// + /// A that tracks before last + /// with the propertyName where the mapped for the have to be set + /// + /// The + /// A collection of that have been mapped during the process of one CSV row + /// If one of the requested during the path build does not exist + private void ProcessPath(PropertyPath path, IReadOnlyList previousThings, IReaderRow reader, List alreadyReadThings, IReadOnlyCollection accessibleThings, Dictionary<(Thing Thing, string PropertyName), List> lastThingsBeforeTargetClassKind, TypeMap typeMap, List mappedThings) + { + var value = QueryValueToUse(reader, path.PropertyMap); + var currentClassKind = path.PropertyMap.ClassKind!.Value; + + var relatedThings = QueryMatchingThings(currentClassKind, value, path.PropertyMap, alreadyReadThings, accessibleThings); + var referencedThings = new List(); + + foreach (var previousThing in previousThings) + { + var referencedValue = previousThing.QueryValue(path.PropertyDescriptor[path.PropertyDescriptor.Depth - 1].Input); + + switch (referencedValue) + { + case Thing tthing: + referencedThings.Add(tthing); + break; + case IEnumerable tthings: + if (path.PropertyMap.FirstOrDefault) + { + if (tthings.FirstOrDefault() is { } firstThing) + { + referencedThings.Add(firstThing); + } + } + else + { + referencedThings.AddRange(tthings); + } + + break; + } + } + + var foundThings = referencedThings.Intersect(relatedThings).ToList(); + alreadyReadThings.AddRange(foundThings); + + if (foundThings.Count == 0) + { + if (currentClassKind != typeMap.ClassKind) + { + this.logger.LogError("The provided CSV references Thing(s) that are not part of the Database: Source : {0}, Value: {1}", path.PropertyMap.Source, value); + throw new InvalidDataException($"The provided CSV references Thing(s) that are not part of the Database: Source : {path.PropertyMap.Source}, Value: {value}"); + } + + var newThing = TypeInitializer.Initialize(currentClassKind); + UpdateThingValues(reader, newThing, typeMap, path.PropertyMap, alreadyReadThings, accessibleThings); + mappedThings.Add(newThing); + + foreach (var previousThing in previousThings) + { + var key = (previousThing, path.PropertyDescriptor[path.PropertyDescriptor.Depth - 1].Input); + + if (lastThingsBeforeTargetClassKind.TryGetValue(key, out var thingsToSet)) + { + thingsToSet.Add(newThing); + } + else + { + lastThingsBeforeTargetClassKind[key] = [newThing]; + } + } + } + else + { + if (currentClassKind == typeMap.ClassKind) + { + foreach (var foundThing in foundThings) + { + UpdateThingValues(reader, foundThing, typeMap, path.PropertyMap, alreadyReadThings, accessibleThings); + mappedThings.Add(foundThing); + } + + foreach (var previousThing in previousThings) + { + var key = (previousThing, path.PropertyDescriptor[path.PropertyDescriptor.Depth - 1].Input); + + if (lastThingsBeforeTargetClassKind.TryGetValue(key, out var thingsToSet)) + { + thingsToSet.AddRange(foundThings); + } + else + { + lastThingsBeforeTargetClassKind[key] = [..foundThings]; + } + } + } + } + + foreach (var propertyPath in path.Children) + { + this.ProcessPath(propertyPath, foundThings, reader, alreadyReadThings, accessibleThings, lastThingsBeforeTargetClassKind, typeMap, mappedThings); + } + } + + /// + /// Queries a collection of for that matches the provided and that have a property value that contains the provided + /// + /// + /// The to match + /// The expected value + /// The that contains information for property name to search on + /// A collection of already read s + /// A of s that comes from the cache + /// The collection of retrieve that matches the request + /// + /// If the is set, every that matches the + /// will be retrieved + /// + private static List QueryMatchingThings(ClassKind classKind, string value, PropertyMap propertyMap, IEnumerable alreadyReadThings, IEnumerable accessibleThings) + { + var allThings = new List(accessibleThings); + allThings.AddRange(alreadyReadThings); + allThings = allThings.Distinct().ToList(); + + return allThings.Where(Predicate).ToList(); + + bool Predicate(Thing thing) + { + return thing.ClassKind == classKind + && (propertyMap.FirstOrDefault || DoesContainsValue(thing.QueryValue(propertyMap.Search), value, propertyMap.Separator)); + } + } + + /// + /// Update all values that have been defined inside the as value setter (when the + /// value is defined) + /// + /// The that provide CSV content read + /// The that where values have to be set + /// The defined + /// + /// The to provide identification to retrieve or create the + /// + /// + /// A collection of already read s + /// A of s that comes from the cache + private static void UpdateThingValues(IReaderRow reader, Thing thingToUpdate, TypeMap typeMap, PropertyMap identifierPropertyMap, IReadOnlyCollection alreadyReadThings, IReadOnlyCollection accessibleThings) + { + if (identifierPropertyMap.Target != null) + { + thingToUpdate.SetValue(identifierPropertyMap.Target, QueryObjectValueToSet(reader, identifierPropertyMap, alreadyReadThings, accessibleThings)); + } + + foreach (var targetPropertyMap in typeMap.Properties.Where(x => !x.IsIdentifierProperty && !string.IsNullOrWhiteSpace(x.Target))) + { + thingToUpdate.SetValue(targetPropertyMap.Target, QueryObjectValueToSet(reader, targetPropertyMap, alreadyReadThings, accessibleThings)); + } + } + + /// + /// Queries the object value that have to be set based on the provided and the content of the CSV + /// + /// The that provide CSV content read + /// The that defines logic + /// A collection of already read s + /// A of s that comes from the cache + /// The object value that have to be set + private static object QueryObjectValueToSet(IReaderRow reader, PropertyMap propertyMap, IReadOnlyCollection alreadyReadThings, IReadOnlyCollection accessibleThings) + { + var value = QueryValueToUse(reader, propertyMap); + + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var splittedValue = value.Split(new[] { propertyMap.Separator }, StringSplitOptions.RemoveEmptyEntries); + + var objects = new List(); + + foreach (var split in splittedValue) + { + if (propertyMap.SearchClassKind.HasValue) + { + bool Predicate(Thing thing) + { + var descriptor = PropertyDescriptor.QueryPropertyDescriptor(propertyMap.Search); + var searchValue = descriptor[descriptor.Depth - 1].Input; + + return thing.ClassKind == propertyMap.SearchClassKind!.Value + && DoesContainsValue(thing.QueryValue(searchValue), split, propertyMap.Separator); + } + + var referencedThings = alreadyReadThings.Where(Predicate).ToList(); + referencedThings.AddRange(accessibleThings.Where(Predicate)); + + objects.AddRange(referencedThings.Distinct()); + } + else + { + objects.Add(split); + } + } + + return objects.Count == 1 ? objects[0] : objects; + } + + /// + /// Validate that the entryPoint is a valid one + /// + /// The + /// The entry point + /// + /// If the is invalid. The is invalid in following cases: + /// - If none entry point is defined + /// - If the is not set + /// - If the is equals to but some identifier + /// are defined + /// + private void ValidateEntryPoint(TypeMap typeMap, PropertyMap entryPoint) + { + var identifierProperties = typeMap.Properties.Where(x => x.IsIdentifierProperty && x != entryPoint); + + if (entryPoint == default) + { + this.logger.LogError("The provided json mapping function does not provide the entry point of the csv file"); + throw new InvalidDataException("The provided json mapping function does not provide the entry point of the csv file"); + } + + if (entryPoint.ClassKind == null) + { + this.logger.LogError("The defined EntryPoint does not specify the ClassKind"); + throw new InvalidDataException("The defined EntryPoint does not specify the ClassKind"); + } + + if (entryPoint.ClassKind == typeMap.ClassKind && identifierProperties.Any()) + { + this.logger.LogError("The provided json mapping is invalid since the entry point is of the same type as the hub type but expect to build a path via other identifiers objects"); + throw new InvalidDataException("The provided json mapping is invalid since the entry point is of the same type as the hub type but expect to build a path via other identifiers objects"); + } + } + + /// + /// Reads the header row from the provided + /// + /// The used to read CSV content + /// The collection of s + /// A + private async Task ReadHeader(IReader reader, IEnumerable typeMaps) + { + await reader.ReadAsync(); + + if (!reader.ReadHeader()) + { + this.logger.LogError("The provided Csv does not provide any header, the mapping cannot continue"); + throw new InvalidOperationException("The provided Csv does not provide any header, the mapping cannot continue"); + } + + var headers = reader.HeaderRecord; + + if (typeMaps.SelectMany(x => x.Properties).FirstOrDefault(p => Array.TrueForAll(headers, x => !x.Equals(p.Source))) is { } invalidPropertyMap) + { + this.logger.LogError("The provided CSV does not contains any header for the source {0}", invalidPropertyMap.Source); + throw new InvalidDataException($"The provided CSV does not contains any header for the source {invalidPropertyMap.Source}"); + } + } + + /// + /// Validates all parameters provided for the method + /// + /// The provided + /// The provided collection of s + /// The provided + /// + /// + private static void ValidateReadParameters(Stream stream, IReadOnlyCollection typeMaps, ISession session) + { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + + if (typeMaps == null) + { + throw new ArgumentNullException(nameof(typeMaps)); + } + + if (session == null) + { + throw new ArgumentNullException(nameof(session)); + } + + if (typeMaps.Count == 0) + { + throw new ArgumentException("The provided collection of TypeMap is empty", nameof(typeMaps)); + } + + if (typeMaps.SelectMany(x => x.Properties).Any(x => x.IsIdentifierProperty && !x.ClassKind.HasValue)) + { + throw new InvalidDataException("One of the IdentifierProperty do not specify the ClassKind to query"); + } + } + + /// + /// Queries the value to use for a cell inside a CSV row based on a + /// + /// The that can retrieve the value of a CSV cell + /// The current + /// The string value to use + /// + /// If the is set, will return , + /// the content of the CSV cell is return in the other case + /// + private static string QueryValueToUse(IReaderRow reader, PropertyMap propertyMap) + { + return propertyMap.SearchBasedOnHeader ? propertyMap.Source : reader.GetField(propertyMap.Source); + } + + /// + /// Verifies that an contains the + /// + /// The where we want to search into + /// An expected string to retrieve + /// A separator + /// The result of the search + private static bool DoesContainsValue(object toSearchInto, string expectedValue, string separator) + { + if (toSearchInto == null) + { + return expectedValue == null; + } + + if (expectedValue.Contains(separator)) + { + var splittedExpectedValue = expectedValue.Split(new[] { separator }, StringSplitOptions.RemoveEmptyEntries); + return Array.Exists(splittedExpectedValue, x => DoesContainsValue(toSearchInto, x, separator)); + } + + if (toSearchInto is IEnumerable enumerable and not string) + { + return enumerable.Cast().Any(value => value.ToString() == expectedValue); + } + + return toSearchInto.ToString() == expectedValue; + } + } +} diff --git a/DEH-CSV/DEH-CSV.csproj b/DEH-CSV/DEH-CSV.csproj index 3c26cee..06bdf17 100644 --- a/DEH-CSV/DEH-CSV.csproj +++ b/DEH-CSV/DEH-CSV.csproj @@ -1,54 +1,43 @@ - - - DEH-CSV - ECSS-E-TM-10-25 to CSV converter - netstandard2.0 - DEH-CSV - CDP COMET ECSS-E-TM-10-25 CSV - cdp4-icon.png - RHEAGROUP.DEHCSV - 1.2.0 - latest - true - true - snupkg - + + DEH-CSV + ECSS-E-TM-10-25 to CSV converter + netstandard2.0 + DEH-CSV + CDP COMET ECSS-E-TM-10-25 CSV + cdp4-icon.png + RHEAGROUP.DEHCSV + 1.2.0 + latest + true + true + snupkg + [Update] to CDP4-SDK version 24.4.0 [Update] to Logging.Abstractions version 8.0.0 - README.md - - - - RHEA System S.A. - Copyright 2023 RHEA System S.A. - Sam Gerené - Apache-2.0 - - - - Git - https://github.com/RHEAGROUP/DEH-CSV - true - - - - - - - - - - - - - - - - - - - - + README.md + + + RHEA System S.A. + Copyright 2023-2024 RHEA System S.A. + Sam Gerené + Apache-2.0 + + + Git + https://github.com/RHEAGROUP/DEH-CSV + true + + + + + + + + + + + + \ No newline at end of file diff --git a/DEH-CSV/Helpers/PropertyPath.cs b/DEH-CSV/Helpers/PropertyPath.cs new file mode 100644 index 0000000..248416e --- /dev/null +++ b/DEH-CSV/Helpers/PropertyPath.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------------------------------------------- +// +// +// Copyright 2023-2024 RHEA System S.A. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// ------------------------------------------------------------------------------------------------- + +namespace RHEAGROUP.DEHCSV.Helpers +{ + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + + using CDP4Common.PropertyAccesor; + + using RHEAGROUP.DEHCSV.Mapping; + + /// + /// The class is used to help building the path to follow based on a collection of association of + /// and + /// + public class PropertyPath + { + /// + /// Initializes a new instance of + /// + /// + /// The collection of that will build the containment of + /// + /// + public PropertyPath(IReadOnlyCollection properties) + { + var allIdentifiers = properties.Where(x => x.IsIdentifierProperty && !string.IsNullOrEmpty(x.Path)) + .Select(x => (PropertyDescriptor.QueryPropertyDescriptor(x.Path), x)); + + var root = properties.Single(x => x.IsIdentifierProperty && string.IsNullOrEmpty(x.Path)); + this.InitializeProperties(null, root, allIdentifiers.ToImmutableList()); + } + + /// + /// Initializes a new + /// + /// The associated + /// The associated + /// + /// The collection of composed with + /// and + /// + protected PropertyPath(PropertyDescriptor descriptor, PropertyMap map, IReadOnlyCollection<(PropertyDescriptor Descriptor, PropertyMap Map)> otherProperties) + { + this.InitializeProperties(descriptor, map, otherProperties); + } + + /// + /// Gets the collection of contained + /// + public IReadOnlyCollection Children { get; private set; } + + /// + /// Gets the associated + /// + public PropertyMap PropertyMap { get; private set; } + + /// + /// Gets the associated + /// + public PropertyDescriptor PropertyDescriptor { get; private set; } + + /// + /// Initializes this properties + /// + /// The associated + /// The associated + /// + /// The collection of composed with + /// and + /// + private void InitializeProperties(PropertyDescriptor descriptor, PropertyMap map, IReadOnlyCollection<(PropertyDescriptor, PropertyMap)> allProperties) + { + this.PropertyMap = map; + this.PropertyDescriptor = descriptor; + this.ComputeChildren(allProperties); + } + + /// + /// Compute the property an initialize contained + /// + /// + /// The collection of composed with + /// and + /// + private void ComputeChildren(IReadOnlyCollection<(PropertyDescriptor Descriptor, PropertyMap Map)> allProperties) + { + var currentDepth = this.PropertyDescriptor?.Depth ?? 0; + var nextProperties = allProperties.Where(x => x.Descriptor.Depth == currentDepth + 1).ToList(); + + if (nextProperties.Count == 1) + { + var (descriptor, map) = nextProperties[0]; + this.Children = new List { new(descriptor, map, allProperties) }; + } + else + { + var propertiesWithSameSource = nextProperties.Where(x => x.Map.Source == this.PropertyMap.Source).ToList(); + var collectionToUseForChildren = propertiesWithSameSource.Count == 0 ? nextProperties : propertiesWithSameSource; + this.Children = new List(collectionToUseForChildren.Select(x => new PropertyPath(x.Descriptor, x.Map, allProperties))); + } + } + } +} diff --git a/DEH-CSV/ICsvReader.cs b/DEH-CSV/ICsvReader.cs new file mode 100644 index 0000000..e5e686b --- /dev/null +++ b/DEH-CSV/ICsvReader.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// +// +// Copyright 2023 RHEA System S.A. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +// ------------------------------------------------------------------------------------------------- + +namespace RHEAGROUP.DEHCSV +{ + using System.Collections.Generic; + using System.IO; + using System.Threading.Tasks; + + using CDP4Common.CommonData; + + using CDP4Dal; + + using RHEAGROUP.DEHCSV.Mapping; + + /// + /// The purpose of the is to read CSV files and transform the content to + /// ECSS-E-TM-10-25 data set based on s + /// + public interface ICsvReader + { + /// + /// Reads the CSV content of the and maps it to s based on the provided collection of + /// s + /// + /// The that contains CSV content + /// The collection of s + /// The that helps to retrieve + /// A that returns a collection of mapped s + Task> Read(Stream stream, IReadOnlyCollection typeMaps, ISession session); + } +} diff --git a/DEH-CSV/Mapping/PropertyMap.cs b/DEH-CSV/Mapping/PropertyMap.cs index 2c6b445..7839b81 100644 --- a/DEH-CSV/Mapping/PropertyMap.cs +++ b/DEH-CSV/Mapping/PropertyMap.cs @@ -1,7 +1,7 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // // -// Copyright 2023 RHEA System S.A. +// Copyright 2023-2024 RHEA System S.A. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,13 +20,16 @@ namespace RHEAGROUP.DEHCSV.Mapping { + using CDP4Common.CommonData; + /// /// The purpose of the is to handle ECSS-E-TM-10-25 to CSV mapping /// public class PropertyMap { /// - /// Gets or sets the source property name on the ECSS-E-TM-10-25 class + /// Gets or sets the source property name on the ECSS-E-TM-10-25 class in case of the mapping from ECSS-E-TM10-25 to CSV. + /// Gets or sets the column name of the CSV to use to retrieve a value /// public string Source { get; set; } @@ -49,5 +52,41 @@ public class PropertyMap /// The default value is the pipe character |. /// public string Separator { get; set; } = "|"; + + /// + /// Gets or sets a value indicating whether the current property targets a value identifying an existing thing or + /// for a thing to be created + /// + public bool IsIdentifierProperty { get; set; } + + /// + /// Gets or sets the path to find Thing(s) + /// + public string Path { get; set; } + + /// + /// Gets or sets the name of property to apply the search filter + /// + public string Search { get; set; } + + /// + /// Gets or sets the that should be retrieved + /// + public ClassKind? ClassKind { get; set; } + + /// + /// Gets or sets the asserts that the search value is the name of the CSV header + /// + public bool SearchBasedOnHeader { get; set; } + + /// + /// Gets or sets the that the search value have to have + /// + public ClassKind? SearchClassKind { get; set; } + + /// + /// Gets or sets the asserts that the object to get is the first or the default based on a reference. + /// + public bool FirstOrDefault { get; set; } } } diff --git a/README.md b/README.md index f002979..ebdcb2e 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ The packages are available on Nuget at https://www.nuget.org/packages/DEH-CSV/ ## Quickstart 1. Install the nuget package into your project or solution. + +### Write + 1. Add a mapping file that states how the properties of an ECSS-E-TM-10-25 [Thing](https://comet-dev-docs.mbsehub.org/) needs to be mapped to fields in a CSV file. 1. Use the [ICsvWriter](https://github.com/RHEAGROUP/DEH-CSV/blob/master/DEH-CSV/ICsvWriter.cs) interface and/or the [CsvWriter](https://github.com/RHEAGROUP/DEH-CSV/blob/master/DEH-CSV/CsvWriter.cs) class. @@ -37,6 +40,25 @@ The `ICsvWriter` interface exposes only one method: **write**. - target: the target directory where the CSV files are to be generated. - options: any kind of `object` that can contain configuration information (this is igonred by the standard implementation but can be used when a derived CsvWriter is created where the `Write` method is overriden). +### Read + + 1. Add a mapping file that states of a CSV file needs to be mapped to the properties of an ECSS-E-TM-10-25 [Thing](https://comet-dev-docs.mbsehub.org/) + 1. Add the corresponding CSV file that needs to be processed + 1. Use the [ICsvReader](https://github.com/RHEAGROUP/DEH-CSV/blob/development/DEH-CSV/ICsvReader.cs) interface and/or the [CsvReader](https://github.com/RHEAGROUP/DEH-CSV/blob/development/DEH-CSV/CsvReader.cs) class. + + The `ICsvReader` interface exposes only one method: **read**. + +``` + public Task> Read(Stream stream, IReadOnlyCollection typeMaps, ISession session); +``` + + - stream: The `Stream` that contains CSV content + - typeMaps: The collection of `TypeMap`s + - session: The `ISession` that helps to retrieve `Thing` + - returns: A `Task` that contains a collection of mapped `Thing`s + + After the read is done, all mapped `Thing`s can be written to the `Isession` by adding them to the correct TopContainer. A manual action will need to be done there, unless a `TypeMap` is created to perform that action. + ## Build Status GitHub actions are used to build and test the libraries