From 4da452705e476b4e15ec86da398188ba61c9b280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rnar=20=C3=98sttveit?= <47412359+bjosttveit@users.noreply.github.com> Date: Wed, 4 Oct 2023 11:11:20 +0200 Subject: [PATCH] Expression validation (#311) * frontend support for expression validation * parse expression validation config * parse config file * evaluate expressions for validation * add argv function * migrate from newtonsoft and fix parsing * improve error handling and fix numeric parse * started making shared tests * improved test runner * fix list of resolved keys * add tests and refactor jsondatamodel * update settings * add app-settings-rewriter to altinn-app-cli * fix upgrade appsettings * refactor JsonDataModel RemoveField and check deleteRow arg * add source and throw exception --- cli-tools/altinn-app-cli/Program.cs | 61 +++- .../AppSettingsRewriter.cs | 110 +++++++ .../Controllers/ResourceController.cs | 17 +- .../Configuration/AppSettings.cs | 26 +- .../Validation/ExpressionValidator.cs | 276 ++++++++++++++++++ .../Features/Validation/ValidationAppSI.cs | 27 +- .../Helpers/DataModel/DataModel.cs | 74 +++++ src/Altinn.App.Core/Helpers/IDataModel.cs | 5 + .../Implementation/AppResourcesSI.cs | 18 +- .../Implementation/DefaultTaskEvents.cs | 4 +- .../Internal/App/IAppResources.cs | 6 + .../Expressions/ExpressionEvaluator.cs | 58 +++- .../Expressions/ExpressionFunctionEnum.cs | 6 +- .../Models/Validation/ExpressionValidation.cs | 41 +++ .../Validation/FrontendSeverityConverter.cs | 42 +++ .../Validation/ValidationIssueSource.cs | 5 + .../Altinn.App.Core.Tests.csproj | 2 + .../Validators/ExpressionValidationTests.cs | 96 ++++++ .../hidden-field.json | 39 +++ .../hidden-page.json | 39 +++ .../many-errors.json | 101 +++++++ .../nested-repeating-hidden-row.json | 179 ++++++++++++ .../nested-repeating-hidden.json | 191 ++++++++++++ .../override.json | 51 ++++ .../repeating-hidden-row.json | 72 +++++ .../repeating-hidden.json | 72 +++++ .../repeating.json | 73 +++++ .../single-field-equals.json | 54 ++++ .../warning.json | 44 +++ .../Helpers/JsonDataModel.cs | 204 +++++++++++-- .../CommonTests/ContextListRoot.cs | 5 +- .../CommonTests/ExpressionTestCaseRoot.cs | 3 +- .../LayoutExpressions/TestDataModel.cs | 10 +- 33 files changed, 1951 insertions(+), 60 deletions(-) create mode 100644 cli-tools/altinn-app-cli/v7Tov8/AppSettingsRewriter/AppSettingsRewriter.cs create mode 100644 src/Altinn.App.Core/Features/Validation/ExpressionValidator.cs create mode 100644 src/Altinn.App.Core/Models/Validation/ExpressionValidation.cs create mode 100644 src/Altinn.App.Core/Models/Validation/FrontendSeverityConverter.cs create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/ExpressionValidationTests.cs create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/hidden-field.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/hidden-page.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/many-errors.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/nested-repeating-hidden-row.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/nested-repeating-hidden.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/override.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/repeating-hidden-row.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/repeating-hidden.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/repeating.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/single-field-equals.json create mode 100644 test/Altinn.App.Core.Tests/Features/Validators/shared-expression-validation-tests/warning.json diff --git a/cli-tools/altinn-app-cli/Program.cs b/cli-tools/altinn-app-cli/Program.cs index 17a257d7a..6402f4d71 100644 --- a/cli-tools/altinn-app-cli/Program.cs +++ b/cli-tools/altinn-app-cli/Program.cs @@ -1,5 +1,7 @@ using System.CommandLine; +using System.CommandLine.Invocation; using System.Reflection; +using altinn_app_cli.v7Tov8.AppSettingsRewriter; using altinn_app_cli.v7Tov8.CodeRewriters; using altinn_app_cli.v7Tov8.ProcessRewriter; using altinn_app_cli.v7Tov8.ProjectChecks; @@ -18,27 +20,42 @@ static async Task Main(string[] args) var projectFolderOption = new Option(name: "--folder", description: "The project folder to read", getDefaultValue: () => "CurrentDirectory"); var projectFileOption = new Option(name: "--project", description: "The project file to read relative to --folder", getDefaultValue: () => "App/App.csproj"); var processFileOption = new Option(name: "--process", description: "The process file to read relative to --folder", getDefaultValue: () => "App/config/process/process.bpmn"); + var appSettingsFolderOption = new Option(name: "--appsettings-folder", description: "The folder where the appsettings.*.json files are located", getDefaultValue: () => "App"); var targetVersionOption = new Option(name: "--target-version", description: "The target version to upgrade to", getDefaultValue: () => "8.0.0-preview.9"); var skipCsprojUpgradeOption = new Option(name: "--skip-csproj-upgrade", description: "Skip csproj file upgrade", getDefaultValue: () => false); var skipCodeUpgradeOption = new Option(name: "--skip-code-upgrade", description: "Skip code upgrade", getDefaultValue: () => false); var skipProcessUpgradeOption = new Option(name: "--skip-process-upgrade", description: "Skip process file upgrade", getDefaultValue: () => false); + var skipAppSettingsUpgradeOption = new Option(name: "--skip-appsettings-upgrade", description: "Skip appsettings file upgrade", getDefaultValue: () => false); var rootCommand = new RootCommand("Command line interface for working with Altinn 3 Applications"); var upgradeCommand = new Command("upgrade", "Upgrade an app from v7 to v8") { projectFolderOption, projectFileOption, processFileOption, + appSettingsFolderOption, targetVersionOption, skipCsprojUpgradeOption, skipCodeUpgradeOption, skipProcessUpgradeOption, + skipAppSettingsUpgradeOption, }; rootCommand.AddCommand(upgradeCommand); var versionCommand = new Command("version", "Print version of altinn-app-cli"); rootCommand.AddCommand(versionCommand); - upgradeCommand.SetHandler(async (projectFolder, projectFile, processFile, targetVersion, skipCodeUpgrade, skipProcessUpgrade, skipCsprojUpgrade) => + upgradeCommand.SetHandler( + async (InvocationContext context) => { + var projectFolder = context.ParseResult.GetValueForOption(projectFolderOption)!; + var projectFile = context.ParseResult.GetValueForOption(projectFileOption)!; + var processFile = context.ParseResult.GetValueForOption(processFileOption)!; + var appSettingsFolder = context.ParseResult.GetValueForOption(appSettingsFolderOption)!; + var targetVersion = context.ParseResult.GetValueForOption(targetVersionOption)!; + var skipCodeUpgrade = context.ParseResult.GetValueForOption(skipCodeUpgradeOption)!; + var skipProcessUpgrade = context.ParseResult.GetValueForOption(skipProcessUpgradeOption)!; + var skipCsprojUpgrade = context.ParseResult.GetValueForOption(skipCsprojUpgradeOption)!; + var skipAppSettingsUpgrade = context.ParseResult.GetValueForOption(skipAppSettingsUpgradeOption)!; + if (projectFolder == "CurrentDirectory") { projectFolder = Directory.GetCurrentDirectory(); @@ -63,11 +80,13 @@ static async Task Main(string[] args) { projectFile = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, projectFile); processFile = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, processFile); + appSettingsFolder = Path.Combine(Directory.GetCurrentDirectory(), projectFolder, appSettingsFolder); } else { projectFile = Path.Combine(projectFolder, projectFile); processFile = Path.Combine(projectFolder, processFile); + appSettingsFolder = Path.Combine(projectFolder, appSettingsFolder); } var projectChecks = new ProjectChecks(projectFile); @@ -77,6 +96,7 @@ static async Task Main(string[] args) returnCode = 2; return; } + if (!skipCsprojUpgrade) { returnCode = await UpgradeNugetVersions(projectFile, targetVersion); @@ -92,6 +112,11 @@ static async Task Main(string[] args) returnCode = await UpgradeProcess(processFile); } + if (!skipAppSettingsUpgrade && returnCode == 0) + { + returnCode = await UpgradeAppSettings(appSettingsFolder); + } + if (returnCode == 0) { Console.WriteLine("Upgrade completed without errors. Please verify that the application is still working as expected."); @@ -100,8 +125,9 @@ static async Task Main(string[] args) { Console.WriteLine("Upgrade completed with errors. Please check for errors in the log above."); } - }, - projectFolderOption, projectFileOption, processFileOption, targetVersionOption, skipCodeUpgradeOption, skipProcessUpgradeOption, skipCsprojUpgradeOption); + } + ); + versionCommand.SetHandler(() => { var version = Assembly.GetEntryAssembly()?.GetCustomAttribute()?.InformationalVersion ?? "Unknown"; @@ -188,4 +214,33 @@ static async Task UpgradeProcess(string processFile) return 0; } + + static async Task UpgradeAppSettings(string appSettingsFolder) + { + if (!Directory.Exists(appSettingsFolder)) + { + Console.WriteLine($"App settings folder {appSettingsFolder} does not exist. Please supply location with --appsettings-folder [path/to/appsettings]"); + return 1; + } + + if (Directory.GetFiles(appSettingsFolder, AppSettingsRewriter.APP_SETTINGS_FILE_PATTERN).Count() == 0) + { + Console.WriteLine($"No appsettings*.json files found in {appSettingsFolder}"); + return 1; + } + + Console.WriteLine("Trying to upgrade appsettings*.json files"); + AppSettingsRewriter rewriter = new(appSettingsFolder); + rewriter.Upgrade(); + await rewriter.Write(); + var warnings = rewriter.GetWarnings(); + foreach (var warning in warnings) + { + Console.WriteLine(warning); + } + + Console.WriteLine(warnings.Any() ? "AppSettings files upgraded with warnings. Review the warnings above and make sure that the appsettings files are still valid." : "AppSettings files upgraded"); + + return 0; + } } diff --git a/cli-tools/altinn-app-cli/v7Tov8/AppSettingsRewriter/AppSettingsRewriter.cs b/cli-tools/altinn-app-cli/v7Tov8/AppSettingsRewriter/AppSettingsRewriter.cs new file mode 100644 index 000000000..860d2b1b3 --- /dev/null +++ b/cli-tools/altinn-app-cli/v7Tov8/AppSettingsRewriter/AppSettingsRewriter.cs @@ -0,0 +1,110 @@ + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace altinn_app_cli.v7Tov8.AppSettingsRewriter; + + +/// +/// Rewrites the appsettings.*.json files +/// +public class AppSettingsRewriter +{ + /// + /// The pattern used to search for appsettings.*.json files + /// + public static readonly string APP_SETTINGS_FILE_PATTERN = "appsettings*.json"; + + private Dictionary appSettingsJsonCollection; + + private readonly IList warnings = new List(); + + /// + /// Initializes a new instance of the class. + /// + public AppSettingsRewriter(string appSettingsFolder) + { + appSettingsJsonCollection = new Dictionary(); + foreach (var file in Directory.GetFiles(appSettingsFolder, APP_SETTINGS_FILE_PATTERN)) + { + var json = File.ReadAllText(file); + var appSettingsJson = JsonNode.Parse(json); + if (appSettingsJson is not JsonObject appSettingsJsonObject) + { + warnings.Add($"Unable to parse AppSettings file {file} as a json object, skipping"); + continue; + } + + this.appSettingsJsonCollection.Add(file, appSettingsJsonObject); + } + } + + /// + /// Gets the warnings + /// + public IList GetWarnings() + { + return warnings; + } + + /// + /// Upgrades the appsettings.*.json files + /// + public void Upgrade() + { + foreach ((var fileName, var appSettingsJson) in appSettingsJsonCollection) + { + RewriteRemoveHiddenDataSetting(fileName, appSettingsJson); + } + } + + /// + /// Writes the appsettings.*.json files + /// + public async Task Write() + { + var tasks = appSettingsJsonCollection.Select(async appSettingsFiles => + { + appSettingsFiles.Deconstruct(out var fileName, out var appSettingsJson); + + JsonSerializerOptions options = new JsonSerializerOptions + { + WriteIndented = true, + }; + await File.WriteAllTextAsync(fileName, appSettingsJson.ToJsonString(options)); + }); + + await Task.WhenAll(tasks); + } + + private void RewriteRemoveHiddenDataSetting(string fileName, JsonObject settings) + { + // Look for "AppSettings" object + settings.TryGetPropertyValue("AppSettings", out var appSettingsNode); + if (appSettingsNode is not JsonObject appSettingsObject) + { + // No "AppSettings" object found, nothing to change + return; + } + + // Look for "RemoveHiddenDataPreview" property + appSettingsObject.TryGetPropertyValue("RemoveHiddenDataPreview", out var removeHiddenDataPreviewNode); + if (removeHiddenDataPreviewNode is not JsonValue removeHiddenDataPreviewValue) + { + // No "RemoveHiddenDataPreview" property found, nothing to change + return; + } + + // Get value of "RemoveHiddenDataPreview" property + if (!removeHiddenDataPreviewValue.TryGetValue(out var removeHiddenDataValue)) + { + warnings.Add($"RemoveHiddenDataPreview has unexpected value {removeHiddenDataPreviewValue.ToJsonString()} in {fileName}, expected a boolean"); + return; + } + + appSettingsObject.Remove("RemoveHiddenDataPreview"); + appSettingsObject.Add("RemoveHiddenData", removeHiddenDataValue); + appSettingsObject.Add("RequiredValidation", removeHiddenDataValue); + + } +} diff --git a/src/Altinn.App.Api/Controllers/ResourceController.cs b/src/Altinn.App.Api/Controllers/ResourceController.cs index df01f78c1..7029e3b5e 100644 --- a/src/Altinn.App.Api/Controllers/ResourceController.cs +++ b/src/Altinn.App.Api/Controllers/ResourceController.cs @@ -265,8 +265,23 @@ public async Task GetFooterLayout(string org, string app) { return NoContent(); } - + return Ok(layout); } + + /// + /// Get validation configuration file. + /// + /// The application owner short name + /// The application name + /// Unique identifier of the model to fetch validations for. + /// The validation configuration file as json. + [HttpGet] + [Route("{org}/{app}/api/validationconfig/{id}")] + public ActionResult GetValidationConfiguration(string org, string app, string id) + { + string? validationConfiguration = _appResourceService.GetValidationConfiguration(id); + return Ok(validationConfiguration); + } } } diff --git a/src/Altinn.App.Core/Configuration/AppSettings.cs b/src/Altinn.App.Core/Configuration/AppSettings.cs index 9974f7960..ab34f341d 100644 --- a/src/Altinn.App.Core/Configuration/AppSettings.cs +++ b/src/Altinn.App.Core/Configuration/AppSettings.cs @@ -23,6 +23,11 @@ public class AppSettings /// public const string JSON_SCHEMA_FILENAME = "schema.json"; + /// + /// Constant for the location of validation configuration file + /// + public const string VALIDATION_CONFIG_FILENAME = "validation.json"; + /// /// The app configuration baseUrl where files are stored in the container /// @@ -83,7 +88,7 @@ public class AppSettings /// public string LayoutSetsFileName { get; set; } = "layout-sets.json"; - /// + /// /// Gets or sets the name of the layout setting file name /// public string FooterFileName { get; set; } = "footer.json"; @@ -103,6 +108,11 @@ public class AppSettings /// public string JsonSchemaFileName { get; set; } = JSON_SCHEMA_FILENAME; + /// + /// Gets or sets The JSON schema file name + /// + public string ValidationConfigurationFileName { get; set; } = VALIDATION_CONFIG_FILENAME; + /// /// Gets or sets the filename for application meta data /// @@ -214,8 +224,18 @@ public string GetResourceFolder() public string AppVersion { get; set; } /// - /// Enable the preview functionality to load layout in backend and remove data from hidden components before validation and task completion + /// Enable the functionality to load layout in backend and remove data from hidden components before task completion + /// + public bool RemoveHiddenData { get; set; } = false; + + /// + /// Enable the functionality to load layout in backend and validate required fields as defined in the layout + /// + public bool RequiredValidation { get; set; } = false; + + /// + /// Enable the functionality to run expression validation in backend /// - public bool RemoveHiddenDataPreview { get; set; } = false; + public bool ExpressionValidation { get; set; } = false; } } diff --git a/src/Altinn.App.Core/Features/Validation/ExpressionValidator.cs b/src/Altinn.App.Core/Features/Validation/ExpressionValidator.cs new file mode 100644 index 000000000..28fbeb047 --- /dev/null +++ b/src/Altinn.App.Core/Features/Validation/ExpressionValidator.cs @@ -0,0 +1,276 @@ +using System.Text.Json; +using Altinn.App.Core.Helpers; +using Altinn.App.Core.Internal.App; +using Altinn.App.Core.Internal.Expressions; +using Altinn.App.Core.Models.Validation; +using Microsoft.Extensions.Logging; + + +namespace Altinn.App.Core.Features.Validation +{ + /// + /// Validates form data against expression validations + /// + public static class ExpressionValidator + { + /// + public static IEnumerable Validate(string dataType, IAppResources appResourceService, IDataModelAccessor dataModel, LayoutEvaluatorState evaluatorState, ILogger logger) + { + var rawValidationConfig = appResourceService.GetValidationConfiguration(dataType); + if (rawValidationConfig == null) + { + // No validation configuration exists for this data type + return new List(); + } + + var validationConfig = JsonDocument.Parse(rawValidationConfig).RootElement; + return Validate(validationConfig, dataModel, evaluatorState, logger); + } + + /// + public static IEnumerable Validate(JsonElement validationConfig, IDataModelAccessor dataModel, LayoutEvaluatorState evaluatorState, ILogger logger) + { + var validationIssues = new List(); + var expressionValidations = ParseExpressionValidationConfig(validationConfig, logger); + foreach (var validationObject in expressionValidations) + { + var baseField = validationObject.Key; + var resolvedFields = dataModel.GetResolvedKeys(baseField); + var validations = validationObject.Value; + foreach (var resolvedField in resolvedFields) + { + var positionalArguments = new[] { resolvedField }; + foreach (var validation in validations) + { + try + { + if (validation.Condition == null) + { + continue; + } + + var isInvalid = ExpressionEvaluator.EvaluateExpression(evaluatorState, validation.Condition, null, positionalArguments); + if (isInvalid is not bool) + { + throw new ArgumentException($"Validation condition for {resolvedField} did not evaluate to a boolean"); + } + if ((bool)isInvalid) + { + var validationIssue = new ValidationIssue + { + Field = resolvedField, + Severity = validation.Severity ?? ValidationIssueSeverity.Error, + CustomTextKey = validation.Message, + Code = validation.Message, + Source = ValidationIssueSources.Expression, + }; + validationIssues.Add(validationIssue); + } + } + catch + { + logger.LogError($"Error while evaluating expression validation for {resolvedField}"); + throw; + } + } + } + } + + + return validationIssues; + } + + private static RawExpressionValidation? ResolveValidationDefinition(string name, JsonElement definition, Dictionary resolvedDefinitions, ILogger logger) + { + var resolvedDefinition = new RawExpressionValidation(); + var rawDefinition = definition.Deserialize(new JsonSerializerOptions + { + ReadCommentHandling = JsonCommentHandling.Skip, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }); + if (rawDefinition == null) + { + logger.LogError($"Validation definition {name} could not be parsed"); + return null; + } + if (rawDefinition.Ref != null) + { + var reference = resolvedDefinitions.GetValueOrDefault(rawDefinition.Ref); + if (reference == null) + { + logger.LogError($"Could not resolve reference {rawDefinition.Ref} for validation {name}"); + return null; + + } + resolvedDefinition.Message = reference.Message; + resolvedDefinition.Condition = reference.Condition; + resolvedDefinition.Severity = reference.Severity; + } + + if (rawDefinition.Message != null) + { + resolvedDefinition.Message = rawDefinition.Message; + } + + if (rawDefinition.Condition != null) + { + resolvedDefinition.Condition = rawDefinition.Condition; + } + + if (rawDefinition.Severity != null) + { + resolvedDefinition.Severity = rawDefinition.Severity; + } + + if (resolvedDefinition.Message == null) + { + logger.LogError($"Validation {name} is missing message"); + return null; + } + + if (resolvedDefinition.Condition == null) + { + logger.LogError($"Validation {name} is missing condition"); + return null; + } + + return resolvedDefinition; + } + + private static ExpressionValidation? ResolveExpressionValidation(string field, JsonElement definition, Dictionary resolvedDefinitions, ILogger logger) + { + + var rawExpressionValidatıon = new RawExpressionValidation(); + + if (definition.ValueKind == JsonValueKind.String) + { + var stringReference = definition.GetString(); + if (stringReference == null) + { + logger.LogError($"Could not resolve null reference for validation for field {field}"); + return null; + } + var reference = resolvedDefinitions.GetValueOrDefault(stringReference); + if (reference == null) + { + logger.LogError($"Could not resolve reference {stringReference} for validation for field {field}"); + return null; + } + rawExpressionValidatıon.Message = reference.Message; + rawExpressionValidatıon.Condition = reference.Condition; + rawExpressionValidatıon.Severity = reference.Severity; + } + else + { + var expressionDefinition = definition.Deserialize(new JsonSerializerOptions + { + ReadCommentHandling = JsonCommentHandling.Skip, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }); + if (expressionDefinition == null) + { + logger.LogError($"Validation for field {field} could not be parsed"); + return null; + } + + if (expressionDefinition.Ref != null) + { + var reference = resolvedDefinitions.GetValueOrDefault(expressionDefinition.Ref); + if (reference == null) + { + logger.LogError($"Could not resolve reference {expressionDefinition.Ref} for validation for field {field}"); + return null; + + } + rawExpressionValidatıon.Message = reference.Message; + rawExpressionValidatıon.Condition = reference.Condition; + rawExpressionValidatıon.Severity = reference.Severity; + } + + if (expressionDefinition.Message != null) + { + rawExpressionValidatıon.Message = expressionDefinition.Message; + } + + if (expressionDefinition.Condition != null) + { + rawExpressionValidatıon.Condition = expressionDefinition.Condition; + } + + if (expressionDefinition.Severity != null) + { + rawExpressionValidatıon.Severity = expressionDefinition.Severity; + } + } + + if (rawExpressionValidatıon.Message == null) + { + logger.LogError($"Validation for field {field} is missing message"); + return null; + } + + if (rawExpressionValidatıon.Condition == null) + { + logger.LogError($"Validation for field {field} is missing condition"); + return null; + } + + var expressionValidation = new ExpressionValidation + { + Message = rawExpressionValidatıon.Message, + Condition = rawExpressionValidatıon.Condition, + Severity = rawExpressionValidatıon.Severity ?? ValidationIssueSeverity.Error, + }; + + return expressionValidation; + } + + private static Dictionary> ParseExpressionValidationConfig(JsonElement expressionValidationConfig, ILogger logger) + { + var expressionValidationDefinitions = new Dictionary(); + JsonElement definitionsObject; + var hasDefinitions = expressionValidationConfig.TryGetProperty("definitions", out definitionsObject); + if (hasDefinitions) + { + foreach (var definitionObject in definitionsObject.EnumerateObject()) + { + var name = definitionObject.Name; + var definition = definitionObject.Value; + var resolvedDefinition = ResolveValidationDefinition(name, definition, expressionValidationDefinitions, logger); + if (resolvedDefinition == null) + { + logger.LogError($"Validation definition {name} could not be resolved"); + continue; + } + expressionValidationDefinitions[name] = resolvedDefinition; + } + } + var expressionValidations = new Dictionary>(); + JsonElement validationsObject; + var hasValidations = expressionValidationConfig.TryGetProperty("validations", out validationsObject); + if (hasValidations) + { + foreach (var validationArray in validationsObject.EnumerateObject()) + { + var field = validationArray.Name; + var validations = validationArray.Value; + foreach (var validation in validations.EnumerateArray()) + { + if (!expressionValidations.ContainsKey(field)) + { + expressionValidations[field] = new List(); + } + var resolvedExpressionValidation = ResolveExpressionValidation(field, validation, expressionValidationDefinitions, logger); + if (resolvedExpressionValidation == null) + { + logger.LogError($"Validation for field {field} could not be resolved"); + continue; + } + expressionValidations[field].Add(resolvedExpressionValidation); + } + } + } + return expressionValidations; + } + } +} diff --git a/src/Altinn.App.Core/Features/Validation/ValidationAppSI.cs b/src/Altinn.App.Core/Features/Validation/ValidationAppSI.cs index 55a85a6a1..cfb9028e1 100644 --- a/src/Altinn.App.Core/Features/Validation/ValidationAppSI.cs +++ b/src/Altinn.App.Core/Features/Validation/ValidationAppSI.cs @@ -1,4 +1,5 @@ using Altinn.App.Core.Configuration; +using Altinn.App.Core.Helpers.DataModel; using Altinn.App.Core.Helpers; using Altinn.App.Core.Internal.App; using Altinn.App.Core.Internal.AppModel; @@ -228,18 +229,32 @@ public async Task> ValidateDataElement(Instance instance, object data = await _dataClient.GetFormData( instanceGuid, modelType, instance.Org, app, instanceOwnerPartyId, Guid.Parse(dataElement.Id)); - if (_appSettings.RemoveHiddenDataPreview) + LayoutEvaluatorState? evaluationState = null; + + // Remove hidden data before validation + if (_appSettings.RequiredValidation || _appSettings.ExpressionValidation) { + var layoutSet = _appResourcesService.GetLayoutSetForTask(dataType.TaskId); - var evaluationState = await _layoutEvaluatorStateInitializer.Init(instance, data, layoutSet?.Id); - // Remove hidden data before validation, set rows to null to preserve indices + evaluationState = await _layoutEvaluatorStateInitializer.Init(instance, data, layoutSet?.Id); LayoutEvaluator.RemoveHiddenData(evaluationState, RowRemovalOption.SetToNull); - // Evaluate expressions in layout and validate that all required data is included and that maxLength - // is respected on groups - var layoutErrors = LayoutEvaluator.RunLayoutValidationsForRequired(evaluationState, dataElement.Id); + } + + // Evaluate expressions in layout and validate that all required data is included and that maxLength + // is respected on groups + if (_appSettings.RequiredValidation) + { + var layoutErrors = LayoutEvaluator.RunLayoutValidationsForRequired(evaluationState!, dataElement.Id); messages.AddRange(layoutErrors); } + // Run expression validations + if (_appSettings.ExpressionValidation) + { + var expressionErrors = ExpressionValidator.Validate(dataType.Id, _appResourcesService, new DataModel(data), evaluationState!, _logger); + messages.AddRange(expressionErrors); + } + // Run Standard mvc validation using the System.ComponentModel.DataAnnotations ModelStateDictionary dataModelValidationResults = new ModelStateDictionary(); var actionContext = new ActionContext( diff --git a/src/Altinn.App.Core/Helpers/DataModel/DataModel.cs b/src/Altinn.App.Core/Helpers/DataModel/DataModel.cs index 9aed92f8d..ec45626c9 100644 --- a/src/Altinn.App.Core/Helpers/DataModel/DataModel.cs +++ b/src/Altinn.App.Core/Helpers/DataModel/DataModel.cs @@ -90,6 +90,80 @@ public DataModel(object serviceModel) return GetModelDataRecursive(keys, index + 1, elementAt, indicies.Length > 0 ? indicies.Slice(1) : indicies); } + /// + public string[] GetResolvedKeys(string key) + { + if (_serviceModel is null) + { + return new string[0]; + } + + var keyParts = key.Split('.'); + return GetResolvedKeysRecursive(keyParts, _serviceModel); + } + + internal static string JoinFieldKeyParts(string? currentKey, string? key) + { + if (String.IsNullOrEmpty(currentKey)) + { + return key ?? ""; + } + if (String.IsNullOrEmpty(key)) + { + return currentKey ?? ""; + } + + return currentKey + "." + key; + } + + private string[] GetResolvedKeysRecursive(string[] keyParts, object currentModel, int currentIndex = 0, string currentKey = "") + { + if (currentModel is null) + { + return new string[0]; + } + + if (currentIndex == keyParts.Length) + { + return new[] { currentKey }; + } + + var (key, groupIndex) = ParseKeyPart(keyParts[currentIndex]); + var prop = currentModel?.GetType().GetProperties().FirstOrDefault(p => IsPropertyWithJsonName(p, key)); + var childModel = prop?.GetValue(currentModel); + if (childModel is null) + { + return new string[0]; + } + + if (childModel is not string && childModel is System.Collections.IEnumerable childModelList) + { + // childModel is an array + if (groupIndex is null) + { + // Index not specified, recurse on all elements + int i = 0; + var resolvedKeys = new List(); + foreach (var child in childModelList) + { + var newResolvedKeys = GetResolvedKeysRecursive(keyParts, child, currentIndex + 1, JoinFieldKeyParts(currentKey, key + "[" + i + "]")); + resolvedKeys.AddRange(newResolvedKeys); + i++; + } + return resolvedKeys.ToArray(); + } + else + { + // Index specified, recurse on that element + return GetResolvedKeysRecursive(keyParts, childModel, currentIndex + 1, JoinFieldKeyParts(currentKey, key + "[" + groupIndex + "]")); + } + } + + // Otherwise, just recurse + return GetResolvedKeysRecursive(keyParts, childModel, currentIndex + 1, JoinFieldKeyParts(currentKey, key)); + + } + private static object? GetElementAt(System.Collections.IEnumerable enumerable, int index) { // Return the element with index = groupIndex (could not find anohter way to get the n'th element in non generic enumerable) diff --git a/src/Altinn.App.Core/Helpers/IDataModel.cs b/src/Altinn.App.Core/Helpers/IDataModel.cs index 2f267da9e..9d720b1aa 100644 --- a/src/Altinn.App.Core/Helpers/IDataModel.cs +++ b/src/Altinn.App.Core/Helpers/IDataModel.cs @@ -26,6 +26,11 @@ public interface IDataModelAccessor /// int? GetModelDataCount(string key, ReadOnlySpan indicies = default); + /// + /// Get all of the resoved keys (including all possible indexes) from a data model key + /// + string[] GetResolvedKeys(string key); + /// /// Return a full dataModelBiding from a context aware binding by adding indicies /// diff --git a/src/Altinn.App.Core/Implementation/AppResourcesSI.cs b/src/Altinn.App.Core/Implementation/AppResourcesSI.cs index 4d5a33dcc..980d1ebfb 100644 --- a/src/Altinn.App.Core/Implementation/AppResourcesSI.cs +++ b/src/Altinn.App.Core/Implementation/AppResourcesSI.cs @@ -119,7 +119,7 @@ public Application GetApplication() Show = applicationMetadata.OnEntry.Show }; } - + return application; } catch (AggregateException ex) @@ -463,5 +463,21 @@ private byte[] ReadFileContentsFromLegalPath(string legalPath, string filePath) return filedata; } + + /// + public string? GetValidationConfiguration(string modelId) + { + string legalPath = $"{_settings.AppBasePath}{_settings.ModelsFolder}"; + string filename = $"{legalPath}{modelId}.{_settings.ValidationConfigurationFileName}"; + PathHelper.EnsureLegalPath(legalPath, filename); + + string? filedata = null; + if (File.Exists(filename)) + { + filedata = File.ReadAllText(filename, Encoding.UTF8); + } + + return filedata; + } } } diff --git a/src/Altinn.App.Core/Implementation/DefaultTaskEvents.cs b/src/Altinn.App.Core/Implementation/DefaultTaskEvents.cs index c5759b312..5e09d3e4c 100644 --- a/src/Altinn.App.Core/Implementation/DefaultTaskEvents.cs +++ b/src/Altinn.App.Core/Implementation/DefaultTaskEvents.cs @@ -165,7 +165,7 @@ public async Task OnEndProcessTask(string endEvent, Instance instance) private async Task RunRemoveHiddenData(Instance instance, Guid instanceGuid, List? dataTypesToLock) { - if (_appSettings?.RemoveHiddenDataPreview == true) + if (_appSettings?.RemoveHiddenData == true) { await RemoveHiddenData(instance, instanceGuid, dataTypesToLock); } @@ -269,7 +269,7 @@ private async Task RemoveHiddenData(Instance instance, Guid instanceGuid, List