Skip to content

Commit

Permalink
Add test framework for the development of the Elm Interactive
Browse files Browse the repository at this point in the history 
Add a dedicated command to support testing the Elm interactive with selected scenarios. Report test execution duration to support the execution engine's development concerning runtime expenses.
  • Loading branch information
@Viir
Viir committed Jan 16, 2022
1 parent aee28cb commit 2128e2d
Show file tree
Hide file tree
Showing 4 changed files with 238 additions and 76 deletions.
98 changes: 98 additions & 0 deletions implement/elm-fullstack/ElmInteractive/TestElmInteractive.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Pine;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;

namespace elm_fullstack.ElmInteractive;

public class TestElmInteractive
{
public record InteractiveScenarioTestResult(
int totalStepsCount,
IReadOnlyList<InteractiveScenarioTestStepResult> testedSteps)
{
public bool Passed => testedSteps.Count == totalStepsCount && testedSteps.All(s => s.Passed) && Exception == null;

public Exception? Exception => testedSteps.Select(s => s.exception).WhereNotNull().FirstOrDefault();
}

public record InteractiveScenarioTestStepResult(
int durationMs,
Exception? exception)
{
public bool Passed => exception == null;
}

static public InteractiveScenarioTestResult TestElmInteractiveScenario(Composition.TreeWithStringPath scenarioTree)
{
var appCodeTree =
scenarioTree.GetNodeAtPath(new[] { "context-app" });

var stepsDirectory =
scenarioTree.GetNodeAtPath(new[] { "steps" });

if (stepsDirectory?.TreeContent == null)
throw new Exception(nameof(stepsDirectory) + " is null");

if (!stepsDirectory.TreeContent.Any())
throw new Exception("Found no stepsDirectories");

using var interactiveSession = new InteractiveSession(appCodeTree: appCodeTree);

var testScenarioSteps = stepsDirectory.TreeContent;

var testedSteps =
testScenarioSteps
.Select(testStep =>
{
var stepStopwatch = System.Diagnostics.Stopwatch.StartNew();

var stepName = testStep.name;

string? submission = null;

try
{
submission =
Encoding.UTF8.GetString(testStep.component.GetBlobAtPath(new[] { "submission" })!.ToArray());

var evalResult =
interactiveSession.SubmitAndGetResultingValue(submission);

var expectedValueFile =
testStep.component.GetBlobAtPath(new[] { "expected-value" });

if (expectedValueFile != null)
{
var expectedValue = Encoding.UTF8.GetString(expectedValueFile.ToArray());

Assert.IsNull(evalResult.Err, "Submission result has error: " + evalResult.Err);

Assert.AreEqual(
expectedValue,
evalResult.Ok?.valueAsElmExpressionText,
"Value from evaluation does not match expected value.");
}

return new InteractiveScenarioTestStepResult(
durationMs: (int)stepStopwatch.Elapsed.TotalMilliseconds,
exception: null);
}
catch (Exception e)
{
return new InteractiveScenarioTestStepResult(
durationMs: (int)stepStopwatch.Elapsed.TotalMilliseconds,
exception: new Exception("Failed step '" + stepName + "' with exception.\nSubmission in this step:\n" + submission, e));
}
})
.TakeUntil(s => !s.Passed)
.ToImmutableList();

return new InteractiveScenarioTestResult(
totalStepsCount: testScenarioSteps.Count,
testedSteps: testedSteps);
}
}
131 changes: 128 additions & 3 deletions implement/elm-fullstack/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace elm_fullstack;

public class Program
{
static public string AppVersionId => "2022-01-11";
static public string AppVersionId => "2022-01-16";

static int AdminInterfaceDefaultPort => 4000;

Expand Down Expand Up @@ -763,6 +763,131 @@ static CommandLineApplication AddInteractiveCmd(CommandLineApplication app) =>
description: "Display additional information to inspect the implementation.",
optionType: CommandOptionType.NoValue);

var testCommand =
enterInteractiveCmd.Command("test", testCmd =>
{
testCmd.Description = "Test the interactive automatically with given scenarios and reports timings.";

var scenarioOption =
testCmd
.Option(
template: "--scenario",
description: "Test scenario which specifies the submissions and can also specify expectations.",
optionType: CommandOptionType.MultipleValue);

testCmd.OnExecute(() =>
{
var consoleForegroundBefore = Console.ForegroundColor;

var scenariosArguments = scenarioOption.Values;

if (0 < scenariosArguments?.Count)
{
Console.WriteLine("Got " + scenariosArguments.Count + " scenario(s) to load...");

var scenariosLoadResults =
scenariosArguments
.ToImmutableDictionary(
testArg => testArg!,
testArg => LoadComposition.LoadFromPathResolvingNetworkDependencies(testArg!).LogToList());

var failedLoads = scenariosLoadResults.Where(r => r.Value.result.Ok.tree == null).ToImmutableList();

if (failedLoads.Any())
{
var failedLoad = failedLoads.First();

Console.WriteLine(
string.Join(
"\n",
"Failed to load scenario from " + failedLoad.Key + ":",
string.Join("\n", failedLoad.Value.log),
failedLoad.Value.result.Err!));

return;
}

var aggregateComposition =
scenariosLoadResults.Count == 1 ?
Composition.FromTreeWithStringPath(scenariosLoadResults.Single().Value.result.Ok.tree) :
Composition.Component.List(
scenariosLoadResults.Select(r => Composition.FromTreeWithStringPath(r.Value.result.Ok.tree)).ToImmutableList());

var aggregateCompositionHash =
CommonConversion.StringBase16FromByteArray(Composition.GetHash(aggregateComposition));

Console.WriteLine(
"Succesfully loaded " + scenariosLoadResults.Count +
" scenario(s) with an aggregate hash of " + aggregateCompositionHash + ".");

var exceptLoadingStopatch = System.Diagnostics.Stopwatch.StartNew();

var scenariosResults =
scenariosLoadResults
.ToImmutableDictionary(
loadResult => loadResult.Key,
loadResult =>
{
var scenarioStopwatch = System.Diagnostics.Stopwatch.StartNew();

var scenarioReport =
ElmInteractive.TestElmInteractive.TestElmInteractiveScenario(loadResult.Value.result.Ok.tree);

return new
{
loadResult = loadResult.Value,
durationMs = scenarioStopwatch.ElapsedMilliseconds,
scenarioReport
};
});

var passedScenarios =
scenariosResults
.Where(t => t.Value.scenarioReport.Passed)
.ToImmutableList();

var failedScenarios =
scenariosResults
.Where(t => !t.Value.scenarioReport.Passed)
.ToImmutableList();

Console.ForegroundColor = failedScenarios.Any() ? ConsoleColor.Red : ConsoleColor.Green;

var overallStats = new[]
{
(label : "Failed", value : failedScenarios.Count.ToString()),
(label : "Passed", value : passedScenarios.Count.ToString()),
(label : "Total", value : scenariosLoadResults.Count.ToString()),
(label : "Duration", value : exceptLoadingStopatch.ElapsedMilliseconds.ToString("### ### ###") + " ms"),
};

Console.WriteLine(
string.Join(
" - ",
(failedScenarios.Any() ? "Failed" : "Passed") + "!",
string.Join(", ", overallStats.Select(stat => stat.label + ": " + stat.value)),
aggregateCompositionHash[..10] + " (elm-fs " + AppVersionId + ")"));

foreach (var failedScenario in failedScenarios)
{
var scenarioId =
CommonConversion.StringBase16FromByteArray(
Composition.GetHash(
Composition.FromTreeWithStringPath(failedScenario.Value.loadResult.result.Ok.tree)));

Console.WriteLine(
"Failed scenario " + scenarioId[..10] + " ('" + failedScenario.Key.Split('\\', '/').LastOrDefault() + "'):");

Console.WriteLine(failedScenario.Value.scenarioReport.Exception?.ToString());
}

Console.ForegroundColor = consoleForegroundBefore;

return;
}
});
});

enterInteractiveCmd.OnExecute(() =>
{
ReadLine.HistoryEnabled = true;
Expand Down Expand Up @@ -1664,8 +1789,8 @@ static public void ReplicateProcessAndLogToConsole(
// https://docs.microsoft.com/en-us/previous-versions//cc723564(v=technet.10)?redirectedfrom=MSDN#XSLTsection127121120120

Console.WriteLine(
"I added the path '" + executableDirectoryPath + "' to the '" + environmentVariableName +
"' environment variable for the current user account. You will be able to use the '" + commandName + "' command in newer instances of the Command Prompt.");
"I added the path '" + executableDirectoryPath + "' to the '" + environmentVariableName +
"' environment variable for the current user account. You will be able to use the '" + commandName + "' command in newer instances of the Command Prompt.");
});

return (executableIsRegisteredOnPath, registerExecutableForCurrentUser);
Expand Down
5 changes: 3 additions & 2 deletions implement/elm-fullstack/elm-fullstack.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
<TargetFramework>net6.0</TargetFramework>
<RootNamespace>elm_fullstack</RootNamespace>
<AssemblyName>elm-fs</AssemblyName>
<AssemblyVersion>2022.0111.0.0</AssemblyVersion>
<FileVersion>2022.0111.0.0</FileVersion>
<AssemblyVersion>2022.0116.0.0</AssemblyVersion>
<FileVersion>2022.0116.0.0</FileVersion>
<Nullable>enable</Nullable>
</PropertyGroup>

Expand All @@ -27,6 +27,7 @@
<PackageReference Include="Microsoft.ClearScript.V8.Native.win-x64" Version="7.1.7" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.0.1" />
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.2.7" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="ReadLine" Version="2.0.1" />
<PackageReference Include="SharpCompress" Version="0.30.0" />
Expand Down
80 changes: 9 additions & 71 deletions implement/test-elm-fullstack/TestElmInteractive.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Pine;
using static elm_fullstack.ElmInteractive.TestElmInteractive;

namespace test_elm_fullstack;

Expand All @@ -13,10 +14,6 @@ public class TestElmInteractive
{
static string pathToScenariosDirectory => @"./../../../elm-interactive-scenarios";

record InteractiveScenarioTestResult(
bool passed,
Exception? exception);

[TestMethod]
public void TestElmInteractiveScenarios()
{
Expand All @@ -26,7 +23,7 @@ public void TestElmInteractiveScenarios()
{
var scenarioName = Path.GetFileName(scenarioDirectory);

if (!Directory.EnumerateFiles(scenarioDirectory, "*", searchOption: SearchOption.AllDirectories).Take(1).Any())
if (!Directory.EnumerateFiles(scenarioDirectory, "*", searchOption: SearchOption.AllDirectories).Any())
{
// Do not stumble over empty directory here. It could be a leftover after git checkout.
continue;
Expand All @@ -36,18 +33,18 @@ public void TestElmInteractiveScenarios()
}

Console.WriteLine("Total scenarios: " + scenariosResults.Count);
Console.WriteLine("Passed: " + scenariosResults.Values.Count(scenarioResult => scenarioResult.passed));
Console.WriteLine("Passed: " + scenariosResults.Values.Count(scenarioResult => scenarioResult.Passed));

var failedScenarios =
scenariosResults
.Where(scenarioNameAndResult => !scenarioNameAndResult.Value.passed)
.Where(scenarioNameAndResult => !scenarioNameAndResult.Value.Passed)
.ToImmutableList();

foreach (var scenarioNameAndResult in failedScenarios)
{
var causeText =
scenarioNameAndResult.Value.exception != null ?
"exception:\n" + scenarioNameAndResult.Value.exception.ToString()
scenarioNameAndResult.Value.Exception != null ?
"exception:\n" + scenarioNameAndResult.Value.Exception.ToString()
:
"unknown cause";

Expand All @@ -62,66 +59,7 @@ public void TestElmInteractiveScenarios()
}
}

static InteractiveScenarioTestResult TestElmInteractiveScenario(string scenarioDirectory)
{
try
{
var appCodeTree =
LoadFromLocalFilesystem.LoadSortedTreeFromPath(Path.Combine(scenarioDirectory, "context-app"));

var stepsDirectories =
Directory.EnumerateDirectories(
Path.Combine(scenarioDirectory, "steps"),
searchPattern: "*",
searchOption: SearchOption.TopDirectoryOnly)
.OrderBy(directory => directory)
.ToImmutableList();

if (!stepsDirectories.Any())
throw new Exception("Found no stepsDirectories");

using (var interactiveSession = new elm_fullstack.ElmInteractive.InteractiveSession(appCodeTree: appCodeTree))
{
foreach (var stepDirectory in stepsDirectories)
{
var stepName = Path.GetFileName(stepDirectory);

string? submission = null;

try
{
submission =
File.ReadAllText(Path.Combine(stepDirectory, "submission"), System.Text.Encoding.UTF8);

var evalResult =
interactiveSession.SubmitAndGetResultingValue(submission);

var expectedValueFilePath = Path.Combine(stepDirectory, "expected-value");

if (File.Exists(expectedValueFilePath))
{
var expectedValue = File.ReadAllText(expectedValueFilePath, System.Text.Encoding.UTF8);

Assert.IsNull(evalResult.Err, "Submission result has error: " + evalResult.Err);

Assert.AreEqual(
expectedValue,
evalResult.Ok?.valueAsElmExpressionText,
"Value from evaluation does not match expected value.");
}
}
catch (Exception e)
{
throw new Exception("Failed step '" + stepName + "' with exception.\nSubmission in this step:\n" + submission, e);
}
}
}

return new InteractiveScenarioTestResult(passed: true, exception: null);
}
catch (Exception e)
{
return new InteractiveScenarioTestResult(passed: false, exception: e);
}
}
InteractiveScenarioTestResult TestElmInteractiveScenario(string scenarioDirectory) =>
elm_fullstack.ElmInteractive.TestElmInteractive.TestElmInteractiveScenario(
LoadFromLocalFilesystem.LoadSortedTreeFromPath(scenarioDirectory)!);
}

0 comments on commit 2128e2d

Please sign in to comment.