Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove JSON examples from artifacts #450

Open
wants to merge 9 commits into
base: dev
Choose a base branch
from
Open

Remove JSON examples from artifacts #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5eeb08b
Remove JSON examples from artifacts
YaSuenag Jan 21, 2024
dd0da3a
Add MakeDir to GSF.CarbonAware.csproj
YaSuenag Jan 21, 2024
8b97f2e
Add MakeDir to CarbonAware.LocationSources.csproj
YaSuenag Jan 21, 2024
a6fae2d
Fix 1-pr.yaml
YaSuenag Jan 22, 2024
8b2dd96
Merge branch 'dev' into pr/remove-jsons
YaSuenag Dec 11, 2024
e9989c1
Fix directory separator
YaSuenag Dec 11, 2024
940ffff
Fix env definition
YaSuenag Dec 11, 2024
576c440
Tweak JsonDataSourceTests
YaSuenag Dec 11, 2024
dffa3da
Make data source directory in JSON DataSource test project
YaSuenag Dec 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .github/workflows/1-pr.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,11 @@ jobs:
directory: ${{ env.DOTNET_SRC_DIR }}

- name: Generate Open API
run: dotnet tool run swagger tofile --output ./wwwroot/api/v1/swagger.yaml --yaml ${{ env.DLL_FILE_PATH }} v1
env:
DataSources__EmissionsDataSource: Json
DataSources__Configurations__Json__Type: JSON
DOTNET_ROLL_FORWARD: LatestMajor
run: dotnet tool run swagger tofile --output ./wwwroot/api/v1/swagger.yaml --yaml ${{ env.DLL_FILE_PATH }} v1
working-directory: ./src/CarbonAware.WebApi/src

- name: Upload swagger artifact
Expand All @@ -146,7 +148,7 @@ jobs:

- name: Docker Run Container
run: |
docker run -d --name runnable-container -p 8080:8080 ca-api
docker run -d --name runnable-container -e DataSources__EmissionsDataSource=Json -e DataSources__Configurations__Json__Type=JSON -p 8080:8080 ca-api
docker container ls

- name: Docker WGET Health Endpoint
Expand Down
10 changes: 9 additions & 1 deletion src/CarbonAware.CLI/test/integrationTests/CarbonAware.CLI.IntegrationTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,12 @@
Include="..\..\..\CarbonAware.DataSources\CarbonAware.DataSources.ElectricityMapsFree\mock\CarbonAware.DataSources.ElectricityMapsFree.Mocks.csproj" />
<ProjectReference Include="..\..\src\CarbonAware.CLI.csproj" />
</ItemGroup>
</Project>

<Target Name="CopyDataFiles" AfterTargets="Build">
<Copy SourceFiles="$(ProjectDir)..\..\..\data\data-sources\test-data-azure-emissions.json"
DestinationFiles="$(TargetDir)\data-sources\json\demo.json" SkipUnchangedFiles="true" />
<Copy SourceFiles="$(ProjectDir)..\..\..\data\location-sources\azure-regions.json"
DestinationFiles="$(TargetDir)\location-sources\json\azure-regions.json" SkipUnchangedFiles="true" />
</Target>

</Project>
1 change: 1 addition & 0 deletions src/CarbonAware.CLI/test/integrationTests/IntegrationTestingBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public void Setup()
{
Environment.SetEnvironmentVariable("DataSources__EmissionsDataSource", "Json");
Environment.SetEnvironmentVariable("DataSources__Configurations__Json__Type", "JSON");
Environment.SetEnvironmentVariable("DataSources__Configurations__Json__DataFileLocation", "test-data-azure-emissions.json");
_dataSourceMocker = new JsonDataSourceMocker();
break;
}
Expand Down
11 changes: 9 additions & 2 deletions src/CarbonAware.DataSources/CarbonAware.DataSources.Json/mock/JsonDataSourceMocker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,18 @@
namespace CarbonAware.DataSources.Json.Mocks;
public class JsonDataSourceMocker : IDataSourceMocker
{
public JsonDataSourceMocker() { }

public string DataFileName { get; set; }

public JsonDataSourceMocker() {
DataFileName = "test-data-azure-emissions.json";
}

public void SetupDataMock(DateTimeOffset start, DateTimeOffset end, string location)
{
string path = new JsonDataSourceConfiguration().DataFileLocation;
var config = new JsonDataSourceConfiguration();
config.DataFileLocation = DataFileName;
string path = config.DataFileLocation;

var data = new List<EmissionsData>();
DateTimeOffset pointTime = start;
Expand Down
13 changes: 5 additions & 8 deletions ...bonAware.DataSources/CarbonAware.DataSources.Json/src/CarbonAware.DataSources.Json.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,6 @@
<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<Content Include="$(ProjectDir)..\..\..\data\data-sources\**\*.json"
Link="\data-sources\json\%(RecursiveDir)%(Filename)%(Extension)">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<InternalsVisibleTo Include="CarbonAware.DataSources.Json.Mocks" />
<InternalsVisibleTo Include="CarbonAware.DataSources.Json.Tests" />
Expand All @@ -24,4 +17,8 @@
<ItemGroup>
<ProjectReference Include="..\..\..\CarbonAware\src\CarbonAware.csproj" />
</ItemGroup>
</Project>

<Target Name="CreateDatasourceDirectory" AfterTargets="Build">
<MakeDir Directories="$(PublishDir)\data-sources\json" />
</Target>
</Project>
2 changes: 0 additions & 2 deletions ...DataSources/CarbonAware.DataSources.Json/src/Configuration/JsonDataSourceConfiguration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ namespace CarbonAware.DataSources.Json.Configuration;
internal class JsonDataSourceConfiguration
{
private const string BaseDirectory = "data-sources/json";
private const string DefaultDataFile = "test-data-azure-emissions.json";
private const string DirectoryRegExPattern = @"^(?!\.{2})[-\\/a-zA-Z_\d\.: ]*$";
private string assemblyDirectory;
private string? dataFileLocation;
Expand All @@ -36,7 +35,6 @@ public JsonDataSourceConfiguration()
{
var assemblyPath = Assembly.GetExecutingAssembly().Location;
assemblyDirectory = Path.GetDirectoryName(assemblyPath)!;
DataFileLocation = DefaultDataFile;
CacheJsonData = true;
}

Expand Down
3 changes: 3 additions & 0 deletions ...e.DataSources/CarbonAware.DataSources.Json/test/CarbonAware.DataSources.Json.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,7 @@
<ProjectReference Include="..\src\CarbonAware.DataSources.Json.csproj" />
</ItemGroup>

<Target Name="CreateDatasourceDirectory" AfterTargets="Build">
<MakeDir Directories="$(TargetDir)\data-sources\json" />
</Target>
</Project>
8 changes: 0 additions & 8 deletions ...onAware.DataSources/CarbonAware.DataSources.Json/test/JsonDataSourceConfigurationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,6 @@ public void Setup()
AssemblyPath = Assembly.GetExecutingAssembly().Location;
}

[Test]
public void GetDefaultDataFileLocation_IsNotNull_ExpectedBaseDir()
{
Assert.That(_configuration.DataFileLocation, Is.Not.Null);
var expectedDir = Path.Combine(Path.GetDirectoryName(AssemblyPath)!, BaseDir);
Assert.That(_configuration.DataFileLocation, Contains.Substring(expectedDir));
}

[TestCase("../newfile.json", TestName = "setting parent's dir")]
[TestCase("~/newfile.json", TestName = "setting user's home dir")]
[TestCase(null, TestName = "setting null filepath")]
Expand Down
5 changes: 2 additions & 3 deletions src/CarbonAware.DataSources/CarbonAware.DataSources.Json/test/JsonDataSourceTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,17 +111,16 @@ public async Task GetCarbonIntensityAsync_ReturnsEmptyEmissionData()
public async Task GetCarbonIntensityAsync_CacheEmissionData(bool cache)
{
var logger = Mock.Of<ILogger<JsonDataSource>>();

var monitor = new Mock<IOptionsMonitor<JsonDataSourceConfiguration>>();
JsonDataSourceMocker dsMocker = new();
var config = new JsonDataSourceConfiguration
{
DataFileLocation = dsMocker.DataFileName,
CacheJsonData = cache
};
monitor.Setup(m => m.CurrentValue).Returns(config);
var dataSource = new JsonDataSource(logger, monitor.Object);

JsonDataSourceMocker dsMocker = new();

var today = DateTimeOffset.Now;
var todayEnd = today.AddMinutes(30);
var todayLocation = new Location() { Name = "japan" };
Expand Down
13 changes: 5 additions & 8 deletions src/CarbonAware.LocationSources/src/CarbonAware.LocationSources.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,13 @@
<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<Content Include="$(ProjectDir)..\..\data\location-sources\**\*.json"
Link="\location-sources\json\%(RecursiveDir)%(Filename)%(Extension)">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<InternalsVisibleTo Include="GSF.CarbonAware" />
<InternalsVisibleTo Include="CarbonAware.LocationSources.Test" />
<InternalsVisibleTo Include="DynamicProxyGenAssembly2" />
</ItemGroup>
</Project>

<Target Name="CreateLocationDirectory" AfterTargets="Build">
<MakeDir Directories="$(PublishDir)\location-sources\json" />
</Target>
</Project>
4 changes: 4 additions & 0 deletions src/CarbonAware.LocationSources/test/CarbonAware.LocationSources.Test.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,8 @@
<ItemGroup>
<ProjectReference Include="..\src\CarbonAware.LocationSources.csproj" />
</ItemGroup>

<Target Name="CreateLocationDirectory" AfterTargets="Build">
<MakeDir Directories="$(TargetDir)\location-sources\json" />
</Target>
</Project>
7 changes: 5 additions & 2 deletions src/CarbonAware.WebApi/src/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS openapi-env
WORKDIR /app
ENV DOTNET_ROLL_FORWARD LatestMajor
COPY . ./
RUN dotnet build CarbonAware.WebApi/src/CarbonAware.WebApi.csproj -o build
RUN dotnet build CarbonAware.WebApi/src/CarbonAware.WebApi.csproj -c Release -o build
WORKDIR /app/CarbonAware.WebApi/src
# Set environment variables for dummy datasource to start "dotnet tool run"
ENV DataSources__EmissionsDataSource=Json
ENV DataSources__Configurations__Json__Type=JSON
RUN dotnet tool restore && \
dotnet tool run swagger tofile --output /app/build/swagger.yaml --yaml /app/build/CarbonAware.WebApi.dll v1

Expand All @@ -17,7 +20,7 @@ ENV DOTNET_ROLL_FORWARD LatestMajor
# Copy everything from source
COPY . ./
# Use implicit restore to build and publish
RUN dotnet publish CarbonAware.WebApi/src/CarbonAware.WebApi.csproj -a $TARGETARCH -o publish
RUN dotnet publish CarbonAware.WebApi/src/CarbonAware.WebApi.csproj -a $TARGETARCH -c Release -o publish


# Build runtime image
Expand Down
10 changes: 0 additions & 10 deletions src/CarbonAware.WebApi/src/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,4 @@
{
"DataSources": {
"EmissionsDataSource": "test-json",
"ForecastDataSource": "", // We don't currently publish a sample test data source for forecasts.
"Configurations": {
"test-json": {
"Type": "JSON",
"DataFileLocation": "test-data-azure-emissions.json"
}
}
},
"Logging": {
"LogLevel": {
"Default": "Information",
Expand Down
4 changes: 3 additions & 1 deletion src/CarbonAware.WebApi/test/integrationTests/CarbonAware.WebApi.IntegrationTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
<Target Name="CopyDataFiles" AfterTargets="Build">
<Copy SourceFiles="$(ProjectDir)..\..\..\data\data-sources\test-data-azure-emissions.json"
DestinationFiles="$(TargetDir)\data-sources\json\demo.json" SkipUnchangedFiles="true" />
<Copy SourceFiles="$(ProjectDir)..\..\..\data\location-sources\azure-regions.json"
DestinationFiles="$(TargetDir)\location-sources\json\azure-regions.json" SkipUnchangedFiles="true" />
</Target>

</Project>
</Project>
2 changes: 1 addition & 1 deletion src/CarbonAware.WebApi/test/integrationTests/CarbonAwareControllerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ namespace CarbonAware.WepApi.IntegrationTests;
[TestFixture(DataSourceType.JSON)]
[TestFixture(DataSourceType.WattTime)]
[TestFixture(DataSourceType.ElectricityMaps)]
[TestFixture(DataSourceType.ElectricityMapsFree)]
//[TestFixture(DataSourceType.ElectricityMapsFree)] // TODO: need to implement data source into IntegrationTestingBase.cs
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to document better why this is being ignored for this PR please?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I updated my Visual Studio Community 2022 to version 17.12.3 , then all tests are passed without commented-out.
スクリーンショット 2025-01-07 204611
This screenshot shows EMFree datasource tests in both CarbonAwareControllerTests.cs and LocationsControllerTests.cs have been skipped. I wonder why they were skipped, but it might be the issue in VS (or test facilities in dotnet) because the issue has gone in the latest VS.

Can someone evaluate on your environment without // ? I will remove // if it works on other environment(s).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The tests failed again!

The root cause is that the test for ElectricityMapsFree would load JsonDataSource.

As I pointed before, ElectricityMapsFree has not been handled at IntegrationTestingBase.
WebAPI testcase would use src/CarbonAware.WebApi/src/appsettings.json which makes dependency to JsonDataSource. Thus tests for ElectricityMapsFree works without errors - it means ElectricityMapsFree tests are not valid because it uses JsonDataSource.

@danuw @vaughanknight
Should we fix the tests for ElectricityMapsFree before this PR?

class CarbonAwareControllerTests : IntegrationTestingBase
{
private readonly string healthURI = "/health";
Expand Down
2 changes: 1 addition & 1 deletion src/CarbonAware.WebApi/test/integrationTests/LocationsControllerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace CarbonAware.WepApi.IntegrationTests;
[TestFixture(DataSourceType.JSON)]
[TestFixture(DataSourceType.WattTime)]
[TestFixture(DataSourceType.ElectricityMaps)]
[TestFixture(DataSourceType.ElectricityMapsFree)]
//[TestFixture(DataSourceType.ElectricityMapsFree)] // TODO: need to implement data source into IntegrationTestingBase.cs
class LocationsControllerTests : IntegrationTestingBase
{
private readonly string locationsURI = "/locations";
Expand Down
8 changes: 4 additions & 4 deletions ...tegrationTests/UnconfiguredWebApiTests.cs → ...t/integrationTests/WebApiEndpointTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
namespace CarbonAware.WebApi.IntegrationTests;

/// <summary>
/// Tests that the Web API starts without configuration.
/// Tests that static Web API endpoints.
/// </summary>
[TestFixture(DataSourceType.None)]
class UnconfiguredWebApiTests : IntegrationTestingBase
[TestFixture(DataSourceType.JSON)]
class WebApiEndpointTests : IntegrationTestingBase
{
private readonly string healthURI = "/health";
private readonly string fakeURI = "/fake-endpoint";

public UnconfiguredWebApiTests(DataSourceType dataSource) : base(dataSource) { }
public WebApiEndpointTests(DataSourceType dataSource) : base(dataSource) { }

[Test]
public async Task HealthCheck_ReturnsOK()
Expand Down
7 changes: 4 additions & 3 deletions src/GSF.CarbonAware/src/GSF.CarbonAware.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,13 @@
<Content Include="$(AssemblyName).targets" Pack="true"
PackagePath="build/$(AssemblyName).targets" />
<Content Include="$(TargetDir)/CarbonAware*.dll" Pack="true" PackagePath="lib/net8.0/" />
<Content Include="$(TargetDir)/data-sources/**/*.json" Pack="true" PackagePath="data-sources/" />
<Content Include="$(TargetDir)/location-sources/**/*.json" Pack="true"
PackagePath="location-sources/" />
</ItemGroup>

<ItemGroup Label="IncludeSymbols" Condition="$(IncludeSymbols) == true">
<Content Include="$(TargetDir)/CarbonAware*.pdb" Pack="true" PackagePath="lib/net8.0/" />
</ItemGroup>

<Target Name="CreateDatasourceDirectory" AfterTargets="Build">
<MakeDir Directories="$(TargetDir)\data-sources\json;$(TargetDir)\location-sources\json" />
</Target>
</Project>
19 changes: 3 additions & 16 deletions src/GSF.CarbonAware/src/GSF.CarbonAware.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,11 @@
<Project>
<ItemGroup>
<DataSourceFiles Include="$(MSBuildThisFileDirectory)/../data-sources/**/*.*" />
</ItemGroup>
<Target Name="CopyDataSourceFiles" BeforeTargets="BeforeBuild">
<Copy SourceFiles="@(DataSourceFiles)" DestinationFolder="$(TargetDir)/data-sources/%(RecursiveDir)" />
<!-- Copy to bin dir for Azure Functions to work -->
<Copy SourceFiles="@(DataSourceFiles)" DestinationFolder="$(TargetDir)/bin/data-sources/%(RecursiveDir)" />
</Target>
<ItemGroup>
<LocationSourceFiles Include="$(MSBuildThisFileDirectory)/../location-sources/**/*.*" />
</ItemGroup>
<Target Name="CopyLocationSourceFiles" BeforeTargets="BeforeBuild">
<Copy SourceFiles="@(LocationSourceFiles)" DestinationFolder="$(TargetDir)/location-sources/%(RecursiveDir)" />
<!-- Copy to bin dir for Azure Functions to work -->
<Copy SourceFiles="@(LocationSourceFiles)" DestinationFolder="$(TargetDir)/bin/location-sources/%(RecursiveDir)" />
</Target>
<ItemGroup>
<DllFiles Include="$(MSBuildThisFileDirectory)/../lib/net8.0/*.dll" />
</ItemGroup>
<Target Name="CopyDllFiles" BeforeTargets="BeforeBuild">
<Copy SourceFiles="@(DllFiles)" DestinationFolder="$(TargetDir)" />
</Target>
<Target Name="CreateDatasourceDirectory" AfterTargets="Build">
<MakeDir Directories="$(PublishDir)\data-sources\json,$(PublishDir)\location-sources\json" />
</Target>
</Project>
Loading