From ab0b9d6b2b3e4e45ac657bfabebf9e166a5481cc Mon Sep 17 00:00:00 2001 From: James Brown <64858662+james-d-brown@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:36:45 +0100 Subject: [PATCH] Allow for the declaration of explicit time pools, #245. --- src/wres/helpers/MainUtilities.java | 4 +- wres-config/nonsrc/schema.yml | 36 +++- .../config/yaml/DeclarationInterpolator.java | 171 ++++++++++++++++++ .../config/yaml/DeclarationUtilities.java | 31 +++- .../config/yaml/DeclarationValidator.java | 34 ++++ .../components/EvaluationDeclaration.java | 13 +- .../ThresholdSourcesDeserializer.java | 3 +- .../deserializers/TimeWindowDeserializer.java | 88 +++++++++ .../config/yaml/DeclarationFactoryTest.java | 112 ++++++++++++ .../yaml/DeclarationInterpolatorTest.java | 66 +++++++ .../config/yaml/DeclarationValidatorTest.java | 26 +++ .../database/TimeSeriesRetriever.java | 21 ++- 12 files changed, 579 insertions(+), 26 deletions(-) create mode 100644 wres-config/src/wres/config/yaml/deserializers/TimeWindowDeserializer.java diff --git a/src/wres/helpers/MainUtilities.java b/src/wres/helpers/MainUtilities.java index b6a78eefaa..c4110ddb88 100644 --- a/src/wres/helpers/MainUtilities.java +++ b/src/wres/helpers/MainUtilities.java @@ -59,8 +59,8 @@ public static boolean isSimpleOperation( String operation ) * @param operation the operation * @return a result that may contain the operation */ - private static Optional>> getOperation( - String operation ) + private static Optional>> + getOperation( String operation ) { Objects.requireNonNull( operation ); String finalOperation = operation.toLowerCase(); diff --git a/wres-config/nonsrc/schema.yml b/wres-config/nonsrc/schema.yml index 6f7da2758b..4852b5cf99 100644 --- a/wres-config/nonsrc/schema.yml +++ b/wres-config/nonsrc/schema.yml @@ -98,16 +98,18 @@ definitions: "$ref": "#/definitions/SpatialMask" reference_dates: "$ref": "#/definitions/Dates" - reference_date_pools: + time_pools: "$ref": "#/definitions/TimePools" + reference_date_pools: + "$ref": "#/definitions/TimePoolSequence" valid_dates: "$ref": "#/definitions/Dates" valid_date_pools: - "$ref": "#/definitions/TimePools" + "$ref": "#/definitions/TimePoolSequence" lead_times: "$ref": "#/definitions/LeadTimes" lead_time_pools: - "$ref": "#/definitions/TimePools" + "$ref": "#/definitions/TimePoolSequence" analysis_times: "$ref": "#/definitions/AnalysisTimes" time_scale: @@ -875,7 +877,33 @@ definitions: - unit TimePools: - title: The temporal boundaries of a pool of data to evaluate. + title: Explicitly declared time pools. + description: "The declaration of an explicit, possibly irregular, sequence + of time pools. Each pool contains an interval of times whose corresponding + pairs will be pooled together when calculating statistics." + type: array + items: + "$ref": "#/definitions/TimePool" + minItems: 1 + uniqueItems: true + additionalProperties: false + + TimePool: + title: Explicitly declared time pool. + description: "The declaration of an explicit time pools. Each pool contains an interval of + times whose corresponding pairs will be pooled together when calculating statistics." + type: object + additionalProperties: false + properties: + reference_dates: + "$ref": "#/definitions/Dates" + valid_dates: + "$ref": "#/definitions/Dates" + lead_times: + "$ref": "#/definitions/LeadTimes" + + TimePoolSequence: + title: The temporal boundaries of a pool sequence to evaluate. description: "The declaration of a regular sequence of time pools. Each pool contains an interval of times whose corresponding pairs will be pooled together when calculating statistics. The regular sequence contains pools diff --git a/wres-config/src/wres/config/yaml/DeclarationInterpolator.java b/wres-config/src/wres/config/yaml/DeclarationInterpolator.java index 314f2605ac..0106fb84e0 100644 --- a/wres-config/src/wres/config/yaml/DeclarationInterpolator.java +++ b/wres-config/src/wres/config/yaml/DeclarationInterpolator.java @@ -3,6 +3,8 @@ import java.net.URI; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collection; @@ -18,6 +20,8 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import com.google.protobuf.Timestamp; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,6 +54,7 @@ import wres.config.yaml.components.ThresholdSourceBuilder; import wres.config.yaml.components.ThresholdType; import wres.config.yaml.components.VariableBuilder; +import wres.statistics.MessageFactory; import wres.statistics.generated.EvaluationStatus; import wres.statistics.generated.EvaluationStatus.EvaluationStatusEvent; import wres.statistics.generated.Geometry; @@ -60,6 +65,7 @@ import wres.statistics.generated.Pool; import wres.statistics.generated.SummaryStatistic; import wres.statistics.generated.TimeScale; +import wres.statistics.generated.TimeWindow; /** *

Interpolates missing declaration from the other declaration present. The interpolation of missing declaration may @@ -170,6 +176,8 @@ public static EvaluationDeclaration interpolate( EvaluationDeclaration declarati DeclarationInterpolator.interpolateMetricParameters( adjustedBuilder ); // Interpolate output formats where none exist DeclarationInterpolator.interpolateOutputFormatsWhenNoneDeclared( adjustedBuilder ); + // Interpolate time windows + DeclarationInterpolator.interpolateTimeWindows( adjustedBuilder ); // Handle any events encountered DeclarationInterpolator.handleEvents( events, notify, true ); @@ -1185,6 +1193,169 @@ private static void interpolateOutputFormatsWhenNoneDeclared( EvaluationDeclarat } } + /** + * Interpolates the missing components of any explicitly declared time windows. + * + * @param builder the builder to mutate + */ + private static void interpolateTimeWindows( EvaluationDeclarationBuilder builder ) + { + Set timeWindows = builder.timeWindows(); + Set adjustedTimeWindows = new HashSet<>(); + + // Set the earliest and latest lead durations + Pair leadTimes = + DeclarationInterpolator.getLeadDurationInterval( builder ); + com.google.protobuf.Duration earliestProtoDuration = leadTimes.getLeft(); + com.google.protobuf.Duration latestProtoDuration = leadTimes.getRight(); + + // Set the earliest and latest valid times + Pair validTimes = DeclarationInterpolator.getValidTimeInterval( builder ); + Timestamp earliestValidProto = validTimes.getLeft(); + Timestamp latestValidProto = validTimes.getRight(); + + // Set the earliest and latest reference times + Pair referenceTimes = DeclarationInterpolator.getReferenceTimeInterval( builder ); + Timestamp earliestReferenceProto = referenceTimes.getLeft(); + Timestamp latestReferenceProto = referenceTimes.getRight(); + + for ( TimeWindow next : timeWindows ) + { + TimeWindow.Builder nextBuilder = next.toBuilder(); + + if ( !next.hasEarliestLeadDuration() ) + { + nextBuilder.setEarliestLeadDuration( earliestProtoDuration ); + } + + if ( !next.hasLatestLeadDuration() ) + { + nextBuilder.setLatestLeadDuration( latestProtoDuration ); + } + + if ( !next.hasEarliestValidTime() ) + { + nextBuilder.setEarliestValidTime( earliestValidProto ); + } + + if ( !next.hasLatestValidTime() ) + { + nextBuilder.setLatestValidTime( latestValidProto ); + } + + if ( !next.hasEarliestReferenceTime() ) + { + nextBuilder.setEarliestReferenceTime( earliestReferenceProto ); + } + + if ( !next.hasLatestReferenceTime() ) + { + nextBuilder.setLatestReferenceTime( latestReferenceProto ); + } + + TimeWindow nextAdjusted = nextBuilder.build(); + adjustedTimeWindows.add( nextAdjusted ); + } + + builder.timeWindows( adjustedTimeWindows ); + } + + /** + * @param builder the declaration builder + * @return the lead duration interval + */ + + private static Pair + getLeadDurationInterval( EvaluationDeclarationBuilder builder ) + { + Duration earliestDuration = MessageFactory.DURATION_MIN; + Duration latestDuration = MessageFactory.DURATION_MAX; + + if ( Objects.nonNull( builder.leadTimes() ) ) + { + if ( Objects.nonNull( builder.leadTimes() + .minimum() ) ) + { + earliestDuration = builder.leadTimes() + .minimum(); + } + if ( Objects.nonNull( builder.leadTimes() + .maximum() ) ) + { + latestDuration = builder.leadTimes() + .maximum(); + } + } + + com.google.protobuf.Duration earliestProtoDuration = MessageFactory.getDuration( earliestDuration ); + com.google.protobuf.Duration latestProtoDuration = MessageFactory.getDuration( latestDuration ); + + return Pair.of( earliestProtoDuration, latestProtoDuration ); + } + + /** + * @param builder the declaration builder + * @return the valid time interval + */ + + private static Pair getValidTimeInterval( EvaluationDeclarationBuilder builder ) + { + Instant earliestValid = Instant.MIN; + Instant latestValid = Instant.MAX; + if ( Objects.nonNull( builder.validDates() ) ) + { + if ( Objects.nonNull( builder.validDates() + .minimum() ) ) + { + earliestValid = builder.validDates() + .minimum(); + } + if ( Objects.nonNull( builder.validDates() + .maximum() ) ) + { + latestValid = builder.validDates() + .maximum(); + } + } + + Timestamp earliestValidProto = MessageFactory.getTimestamp( earliestValid ); + Timestamp latestValidProto = MessageFactory.getTimestamp( latestValid ); + + return Pair.of( earliestValidProto, latestValidProto ); + } + + + /** + * @param builder the declaration builder + * @return the reference time interval + */ + + private static Pair getReferenceTimeInterval( EvaluationDeclarationBuilder builder ) + { + Instant earliestReference = Instant.MIN; + Instant latestReference = Instant.MAX; + if ( Objects.nonNull( builder.referenceDates() ) ) + { + if ( Objects.nonNull( builder.referenceDates() + .minimum() ) ) + { + earliestReference = builder.referenceDates() + .minimum(); + } + if ( Objects.nonNull( builder.referenceDates() + .maximum() ) ) + { + latestReference = builder.referenceDates() + .maximum(); + } + } + + Timestamp earliestReferenceProto = MessageFactory.getTimestamp( earliestReference ); + Timestamp latestReferenceProto = MessageFactory.getTimestamp( latestReference ); + + return Pair.of( earliestReferenceProto, latestReferenceProto ); + } + /** * Adds the unit to each value threshold in the set that does not have the unit defined. * diff --git a/wres-config/src/wres/config/yaml/DeclarationUtilities.java b/wres-config/src/wres/config/yaml/DeclarationUtilities.java index 12ae787757..136bb2971e 100644 --- a/wres-config/src/wres/config/yaml/DeclarationUtilities.java +++ b/wres-config/src/wres/config/yaml/DeclarationUtilities.java @@ -121,7 +121,9 @@ public static Set getTimeWindows( EvaluationDeclaration declaration TimePools referenceDatesPools = declaration.referenceDatePools(); TimePools validDatesPools = declaration.validDatePools(); - // Has explicit pooling windows + Set timeWindows; + + // Add the time windows generated from a declared sequence if ( Objects.nonNull( leadDurationPools ) || Objects.nonNull( referenceDatesPools ) || Objects.nonNull( validDatesPools ) ) @@ -132,49 +134,49 @@ public static Set getTimeWindows( EvaluationDeclaration declaration { LOGGER.debug( "Building time windows for reference dates and valid dates and lead durations." ); - return DeclarationUtilities.getReferenceDatesValidDatesAndLeadDurationTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getReferenceDatesValidDatesAndLeadDurationTimeWindows( declaration ); } // Reference dates and valid dates else if ( Objects.nonNull( referenceDatesPools ) && Objects.nonNull( validDatesPools ) ) { LOGGER.debug( "Building time windows for reference dates and valid dates." ); - return DeclarationUtilities.getReferenceDatesAndValidDatesTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getReferenceDatesAndValidDatesTimeWindows( declaration ); } // Reference dates and lead durations else if ( Objects.nonNull( referenceDatesPools ) && Objects.nonNull( leadDurationPools ) ) { LOGGER.debug( "Building time windows for reference dates and lead durations." ); - return DeclarationUtilities.getReferenceDatesAndLeadDurationTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getReferenceDatesAndLeadDurationTimeWindows( declaration ); } // Valid dates and lead durations else if ( Objects.nonNull( validDatesPools ) && Objects.nonNull( leadDurationPools ) ) { LOGGER.debug( "Building time windows for valid dates and lead durations." ); - return DeclarationUtilities.getValidDatesAndLeadDurationTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getValidDatesAndLeadDurationTimeWindows( declaration ); } // Reference dates else if ( Objects.nonNull( referenceDatesPools ) ) { LOGGER.debug( "Building time windows for reference dates." ); - return DeclarationUtilities.getReferenceDatesTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getReferenceDatesTimeWindows( declaration ); } // Lead durations else if ( Objects.nonNull( leadDurationPools ) ) { LOGGER.debug( "Building time windows for lead durations." ); - return DeclarationUtilities.getLeadDurationTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getLeadDurationTimeWindows( declaration ); } // Valid dates else { LOGGER.debug( "Building time windows for valid dates." ); - return DeclarationUtilities.getValidDatesTimeWindows( declaration ); + timeWindows = DeclarationUtilities.getValidDatesTimeWindows( declaration ); } } // One big pool @@ -182,8 +184,19 @@ else if ( Objects.nonNull( leadDurationPools ) ) { LOGGER.debug( "Building one big time window." ); - return Collections.singleton( DeclarationUtilities.getOneBigTimeWindow( declaration ) ); + timeWindows = Collections.singleton( DeclarationUtilities.getOneBigTimeWindow( declaration ) ); } + + // Add the explicitly declared time windows + Set finalWindows = new HashSet<>( timeWindows ); + + LOGGER.debug( "Added {} explicitly declared time pools to the overall group of time pools.", + declaration.timeWindows() + .size() ); + + finalWindows.addAll( declaration.timeWindows() ); + + return Collections.unmodifiableSet( finalWindows ); } /** diff --git a/wres-config/src/wres/config/yaml/DeclarationValidator.java b/wres-config/src/wres/config/yaml/DeclarationValidator.java index c67cbc7b3c..12cbf06f16 100644 --- a/wres-config/src/wres/config/yaml/DeclarationValidator.java +++ b/wres-config/src/wres/config/yaml/DeclarationValidator.java @@ -2194,6 +2194,40 @@ private static List timePoolsAreValid( EvaluationDeclarat declaration.leadTimes() ); events.addAll( leadTimePools ); + // Add a warning if a pool sequence is combined with explicitly generated pools + Set generated = new TreeSet<>(); + if ( Objects.nonNull( declaration.validDatePools() ) ) + { + generated.add( "'valid_date_pools'" ); + } + if ( Objects.nonNull( declaration.referenceDatePools() ) ) + { + generated.add( "'reference_date_pools'" ); + } + if ( Objects.nonNull( declaration.leadTimePools() ) ) + { + generated.add( "'lead_time_pools'" ); + } + + if ( !declaration.timeWindows() + .isEmpty() + && !generated.isEmpty() ) + { + EvaluationStatusEvent warning + = EvaluationStatusEvent.newBuilder() + .setStatusLevel( StatusLevel.WARN ) + .setEventMessage( "The declaration contained both an explicit list of " + + "'time_pools' and an implicitly declared sequence of " + + "pools (" + + generated + + "). This is allowed and the " + + "resulting pools from all sources will be added " + + "together. If this was not intended, please adjust " + + "your declaration." ) + .build(); + events.add( warning ); + } + return Collections.unmodifiableList( events ); } diff --git a/wres-config/src/wres/config/yaml/components/EvaluationDeclaration.java b/wres-config/src/wres/config/yaml/components/EvaluationDeclaration.java index b264be78ad..2678721b0e 100644 --- a/wres-config/src/wres/config/yaml/components/EvaluationDeclaration.java +++ b/wres-config/src/wres/config/yaml/components/EvaluationDeclaration.java @@ -24,6 +24,7 @@ import wres.config.yaml.deserializers.ThresholdSetsDeserializer; import wres.config.yaml.deserializers.ThresholdSourcesDeserializer; import wres.config.yaml.deserializers.ThresholdsDeserializer; +import wres.config.yaml.deserializers.TimeWindowDeserializer; import wres.config.yaml.serializers.ChronoUnitSerializer; import wres.config.yaml.serializers.CrossPairSerializer; import wres.config.yaml.serializers.DecimalFormatSerializer; @@ -34,6 +35,7 @@ import wres.config.yaml.serializers.ThresholdsSerializer; import wres.statistics.generated.Pool; import wres.statistics.generated.SummaryStatistic; +import wres.statistics.generated.TimeWindow; /** * Root class for an evaluation declaration. @@ -81,13 +83,15 @@ public record EvaluationDeclaration( @JsonProperty( "label" ) String label, @JsonProperty( "observed" ) Dataset left, @JsonProperty( "predicted" ) Dataset right, @JsonProperty( "baseline" ) BaselineDataset baseline, - @JsonProperty( "covariates") List covariates, + @JsonProperty( "covariates" ) List covariates, @JsonProperty( "features" ) Features features, @JsonProperty( "feature_groups" ) FeatureGroups featureGroups, @JsonProperty( "feature_service" ) FeatureService featureService, @JsonProperty( "spatial_mask" ) SpatialMask spatialMask, @JsonProperty( "unit" ) String unit, @JsonProperty( "unit_aliases" ) Set unitAliases, + @JsonDeserialize( using = TimeWindowDeserializer.class ) + @JsonProperty( "time_pools" ) Set timeWindows, @JsonProperty( "reference_dates" ) TimeInterval referenceDates, @JsonProperty( "reference_date_pools" ) TimePools referenceDatePools, @JsonProperty( "valid_dates" ) TimeInterval validDates, @@ -195,6 +199,11 @@ public record EvaluationDeclaration( @JsonProperty( "label" ) String label, unitAliases = this.emptyOrUnmodifiableSet( unitAliases, "unit aliases" ); } + if ( Objects.isNull( timeWindows ) ) + { + timeWindows = this.emptyOrUnmodifiableSet( timeWindows, "time pools" ); + } + if ( Objects.isNull( rescaleLenience ) ) { rescaleLenience = TimeScaleLenience.NONE; @@ -240,7 +249,7 @@ public record EvaluationDeclaration( @JsonProperty( "label" ) String label, minimumSampleSize = 0; } - if( Objects.isNull( covariates ) ) + if ( Objects.isNull( covariates ) ) { covariates = List.of(); } diff --git a/wres-config/src/wres/config/yaml/deserializers/ThresholdSourcesDeserializer.java b/wres-config/src/wres/config/yaml/deserializers/ThresholdSourcesDeserializer.java index cfe86569e5..d85a4860b4 100644 --- a/wres-config/src/wres/config/yaml/deserializers/ThresholdSourcesDeserializer.java +++ b/wres-config/src/wres/config/yaml/deserializers/ThresholdSourcesDeserializer.java @@ -80,10 +80,9 @@ else if ( node instanceof TextNode text ) * Creates a threshold source from a plain URI node. * @param plainNode the plain node * @return the threshold source - * @throws IOException if the source could not be read for any reason */ - private ThresholdSource getPlainSource( TextNode plainNode ) throws IOException + private ThresholdSource getPlainSource( TextNode plainNode ) { String uriString = plainNode.textValue(); LOGGER.debug( "Encountered a simple threshold source containing a URI: {}", uriString ); diff --git a/wres-config/src/wres/config/yaml/deserializers/TimeWindowDeserializer.java b/wres-config/src/wres/config/yaml/deserializers/TimeWindowDeserializer.java new file mode 100644 index 0000000000..7d46b90ff2 --- /dev/null +++ b/wres-config/src/wres/config/yaml/deserializers/TimeWindowDeserializer.java @@ -0,0 +1,88 @@ +package wres.config.yaml.deserializers; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectReader; +import com.google.protobuf.Timestamp; + +import wres.config.yaml.components.LeadTimeInterval; +import wres.config.yaml.components.TimeInterval; +import wres.statistics.MessageFactory; +import wres.statistics.generated.TimeWindow; + +/** + * Custom deserializer for a {@link TimeWindow}. + * + * @author James Brown + */ +public class TimeWindowDeserializer extends JsonDeserializer> +{ + @Override + public Set deserialize( JsonParser jp, DeserializationContext context ) + throws IOException + { + Set timeWindows = new HashSet<>(); + + ObjectReader mapper = ( ObjectReader ) jp.getCodec(); + JsonNode node = mapper.readTree( jp ); + + if ( node.isArray() ) + { + int count = node.size(); + for ( int i = 0; i < count; i++ ) + { + TimeWindow.Builder builder = TimeWindow.newBuilder(); + + JsonNode nextNode = node.get( i ); + + // Valid dates + if ( nextNode.has( "valid_dates" ) ) + { + JsonNode validDatesNode = nextNode.get( "valid_dates" ); + TimeInterval validDates = mapper.readValue( validDatesNode, TimeInterval.class ); + + Timestamp minimum = MessageFactory.getTimestamp( validDates.minimum() ); + Timestamp maximum = MessageFactory.getTimestamp( validDates.maximum() ); + builder.setEarliestValidTime( minimum ) + .setLatestValidTime( maximum ); + } + + // Reference dates + if ( nextNode.has( "reference_dates" ) ) + { + JsonNode referenceDatesNode = nextNode.get( "reference_dates" ); + TimeInterval referenceDates = mapper.readValue( referenceDatesNode, TimeInterval.class ); + + Timestamp minimum = MessageFactory.getTimestamp( referenceDates.minimum() ); + Timestamp maximum = MessageFactory.getTimestamp( referenceDates.maximum() ); + builder.setEarliestReferenceTime( minimum ) + .setLatestReferenceTime( maximum ); + } + + // Lead times? + if ( nextNode.has( "lead_times" ) ) + { + JsonNode leadTimesNode = nextNode.get( "lead_times" ); + LeadTimeInterval leadTimes = mapper.readValue( leadTimesNode, LeadTimeInterval.class ); + + com.google.protobuf.Duration minimum = MessageFactory.getDuration( leadTimes.minimum() ); + com.google.protobuf.Duration maximum = MessageFactory.getDuration( leadTimes.maximum() ); + builder.setEarliestLeadDuration( minimum ) + .setLatestLeadDuration( maximum ); + } + + TimeWindow nextWindow = builder.build(); + timeWindows.add( nextWindow ); + } + } + + return Collections.unmodifiableSet( timeWindows ); + } +} \ No newline at end of file diff --git a/wres-config/test/wres/config/yaml/DeclarationFactoryTest.java b/wres-config/test/wres/config/yaml/DeclarationFactoryTest.java index 760996ef94..4d6897a945 100644 --- a/wres-config/test/wres/config/yaml/DeclarationFactoryTest.java +++ b/wres-config/test/wres/config/yaml/DeclarationFactoryTest.java @@ -80,6 +80,7 @@ import wres.config.yaml.components.Values; import wres.config.yaml.components.Variable; import wres.config.yaml.components.VariableBuilder; +import wres.statistics.MessageFactory; import wres.statistics.generated.Geometry; import wres.statistics.generated.GeometryGroup; import wres.statistics.generated.GeometryTuple; @@ -88,6 +89,7 @@ import wres.statistics.generated.SummaryStatistic; import wres.statistics.generated.Threshold; import wres.statistics.generated.TimeScale; +import wres.statistics.generated.TimeWindow; /** * Tests the {@link DeclarationFactory}. @@ -2275,6 +2277,116 @@ void testDeserializeWithCovariates() throws IOException assertEquals( expected, actual ); } + @Test + void testDeserializeWithExplicitTimePools() throws IOException + { + String yaml = """ + observed: + - some_file.csv + predicted: + sources: another_file.csv + time_pools: + - lead_times: + minimum: 1 + maximum: 6 + unit: hours + reference_dates: + minimum: 2551-03-17T00:00:00Z + maximum: 2551-03-20T00:00:00Z + valid_dates: + minimum: 2551-03-18T00:00:00Z + maximum: 2551-03-21T00:00:00Z + - lead_times: + minimum: 7 + maximum: 12 + unit: hours + reference_dates: + minimum: 2551-03-21T00:00:00Z + maximum: 2551-03-23T00:00:00Z + valid_dates: + minimum: 2551-03-22T00:00:00Z + maximum: 2551-03-24T00:00:00Z + """; + + EvaluationDeclaration actual = DeclarationFactory.from( yaml ); + + Instant expectedInstantOne = Instant.parse( "2551-03-17T00:00:00Z" ); + Instant expectedInstantTwo = Instant.parse( "2551-03-18T00:00:00Z" ); + Instant expectedInstantThree = Instant.parse( "2551-03-20T00:00:00Z" ); + Instant expectedInstantFour = Instant.parse( "2551-03-21T00:00:00Z" ); + Instant expectedInstantFive = Instant.parse( "2551-03-22T00:00:00Z" ); + Instant expectedInstantSix = Instant.parse( "2551-03-23T00:00:00Z" ); + Instant expectedInstantSeven = Instant.parse( "2551-03-24T00:00:00Z" ); + + java.time.Duration expectedDurationOne = java.time.Duration.ofHours( 1 ); + java.time.Duration expectedDurationTwo = java.time.Duration.ofHours( 6 ); + java.time.Duration expectedDurationThree = java.time.Duration.ofHours( 7 ); + java.time.Duration expectedDurationFour = java.time.Duration.ofHours( 12 ); + + TimeWindow expectedOne = TimeWindow.newBuilder() + .setEarliestValidTime( MessageFactory.getTimestamp( expectedInstantTwo ) ) + .setLatestValidTime( MessageFactory.getTimestamp( expectedInstantFour ) ) + .setEarliestReferenceTime( MessageFactory.getTimestamp( expectedInstantOne ) ) + .setLatestReferenceTime( MessageFactory.getTimestamp( expectedInstantThree ) ) + .setEarliestLeadDuration( MessageFactory.getDuration( expectedDurationOne ) ) + .setLatestLeadDuration( MessageFactory.getDuration( expectedDurationTwo ) ) + .build(); + + TimeWindow expectedTwo = TimeWindow.newBuilder() + .setEarliestValidTime( MessageFactory.getTimestamp( expectedInstantFive ) ) + .setLatestValidTime( MessageFactory.getTimestamp( expectedInstantSeven ) ) + .setEarliestReferenceTime( MessageFactory.getTimestamp( expectedInstantFour ) ) + .setLatestReferenceTime( MessageFactory.getTimestamp( expectedInstantSix ) ) + .setEarliestLeadDuration( MessageFactory.getDuration( expectedDurationThree ) ) + .setLatestLeadDuration( MessageFactory.getDuration( expectedDurationFour ) ) + .build(); + + Set expected = Set.of( expectedOne, expectedTwo ); + + assertEquals( expected, actual.timeWindows() ); + } + + @Test + void testDeserializeWithExplicitTimePoolsThatAreSparselyDeclared() throws IOException + { + String yaml = """ + observed: + - some_file.csv + predicted: + sources: another_file.csv + time_pools: + - reference_dates: + minimum: 2551-03-17T00:00:00Z + maximum: 2551-03-20T00:00:00Z + - lead_times: + minimum: 7 + maximum: 12 + unit: hours + """; + + EvaluationDeclaration actual = DeclarationFactory.from( yaml ); + + Instant expectedInstantOne = Instant.parse( "2551-03-17T00:00:00Z" ); + Instant expectedInstantTwo = Instant.parse( "2551-03-20T00:00:00Z" ); + + java.time.Duration expectedDurationOne = java.time.Duration.ofHours( 7 ); + java.time.Duration expectedDurationTwo = java.time.Duration.ofHours( 12 ); + + TimeWindow expectedOne = TimeWindow.newBuilder() + .setEarliestReferenceTime( MessageFactory.getTimestamp( expectedInstantOne ) ) + .setLatestReferenceTime( MessageFactory.getTimestamp( expectedInstantTwo ) ) + .build(); + + TimeWindow expectedTwo = TimeWindow.newBuilder() + .setEarliestLeadDuration( MessageFactory.getDuration( expectedDurationOne ) ) + .setLatestLeadDuration( MessageFactory.getDuration( expectedDurationTwo ) ) + .build(); + + Set expected = Set.of( expectedOne, expectedTwo ); + + assertEquals( expected, actual.timeWindows() ); + } + @Test void testDeserializeThrowsExpectedExceptionWhenDuplicateKeysEncountered() { diff --git a/wres-config/test/wres/config/yaml/DeclarationInterpolatorTest.java b/wres-config/test/wres/config/yaml/DeclarationInterpolatorTest.java index f05194af9a..acfbfb2cab 100644 --- a/wres-config/test/wres/config/yaml/DeclarationInterpolatorTest.java +++ b/wres-config/test/wres/config/yaml/DeclarationInterpolatorTest.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.URI; import java.time.Duration; +import java.time.Instant; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collections; @@ -41,6 +42,7 @@ import wres.config.yaml.components.Features; import wres.config.yaml.components.FeaturesBuilder; import wres.config.yaml.components.Formats; +import wres.config.yaml.components.LeadTimeInterval; import wres.config.yaml.components.LeadTimeIntervalBuilder; import wres.config.yaml.components.Metric; import wres.config.yaml.components.MetricBuilder; @@ -53,7 +55,9 @@ import wres.config.yaml.components.ThresholdSource; import wres.config.yaml.components.ThresholdSourceBuilder; import wres.config.yaml.components.ThresholdType; +import wres.config.yaml.components.TimeInterval; import wres.config.yaml.components.VariableBuilder; +import wres.statistics.MessageFactory; import wres.statistics.generated.Geometry; import wres.statistics.generated.GeometryGroup; import wres.statistics.generated.GeometryTuple; @@ -62,6 +66,7 @@ import wres.statistics.generated.SummaryStatistic; import wres.statistics.generated.Threshold; import wres.statistics.generated.TimeScale; +import wres.statistics.generated.TimeWindow; /** * Tests the {@link DeclarationFactory}. @@ -1606,6 +1611,67 @@ void testInterpolateInconsistentTimeScaleProducesError() .contains( "The declared evaluation timescale" ) ); } + @Test + void testInterpolateMissingTimeWindowComponents() + { + Instant validWindowOne = Instant.parse( "2551-03-17T00:00:00Z" ); + Instant validWindowTwo = Instant.parse( "2551-03-20T00:00:00Z" ); + + Duration durationOne = Duration.ofHours( 7 ); + Duration durationTwo = Duration.ofHours( 12 ); + + TimeWindow one = TimeWindow.newBuilder() + .setEarliestValidTime( MessageFactory.getTimestamp( validWindowOne ) ) + .setLatestValidTime( MessageFactory.getTimestamp( validWindowTwo ) ) + .build(); + + TimeWindow two = TimeWindow.newBuilder() + .setEarliestLeadDuration( MessageFactory.getDuration( durationOne ) ) + .setLatestLeadDuration( MessageFactory.getDuration( durationTwo ) ) + .build(); + + Set timeWindows = Set.of( one, two ); + + Instant validOne = Instant.parse( "2033-12-01T09:15:23Z" ); + Instant validTwo = Instant.parse( "2083-12-01T09:15:23Z" ); + + EvaluationDeclaration declaration = + EvaluationDeclarationBuilder.builder() + .left( this.observedDataset ) + .right( this.predictedDataset ) + .leadTimes( new LeadTimeInterval( Duration.ofHours( 1 ), + Duration.ofHours( 100 ) ) ) + .validDates( new TimeInterval( validOne, validTwo ) ) + .timeWindows( timeWindows ) + .build(); + + EvaluationDeclaration interpolated = DeclarationInterpolator.interpolate( declaration, false ); + + Set actual = interpolated.timeWindows(); + + TimeWindow expectedOne = TimeWindow.newBuilder() + .setEarliestValidTime( MessageFactory.getTimestamp( validWindowOne ) ) + .setLatestValidTime( MessageFactory.getTimestamp( validWindowTwo ) ) + .setEarliestReferenceTime( MessageFactory.getTimestamp( Instant.MIN ) ) + .setLatestReferenceTime( MessageFactory.getTimestamp( Instant.MAX ) ) + .setEarliestLeadDuration( MessageFactory.getDuration( Duration.ofHours( 1 ) ) ) + .setLatestLeadDuration( MessageFactory.getDuration( Duration.ofHours( 100 ) ) ) + .build(); + + TimeWindow expectedTwo = TimeWindow.newBuilder() + .setEarliestValidTime( MessageFactory.getTimestamp( validOne ) ) + .setLatestValidTime( MessageFactory.getTimestamp( validTwo ) ) + .setEarliestReferenceTime( MessageFactory.getTimestamp( Instant.MIN ) ) + .setLatestReferenceTime( MessageFactory.getTimestamp( Instant.MAX ) ) + .setEarliestLeadDuration( MessageFactory.getDuration( durationOne ) ) + .setLatestLeadDuration( MessageFactory.getDuration( durationTwo ) ) + .build(); + + Set expected = Set.of( expectedOne, expectedTwo ); + + assertEquals( expected, actual ); + } + // The testDeserializeAndInterpolate* tests are integration tests of deserialization plus interpolation @Test diff --git a/wres-config/test/wres/config/yaml/DeclarationValidatorTest.java b/wres-config/test/wres/config/yaml/DeclarationValidatorTest.java index d5e6e7551f..64dc124763 100644 --- a/wres-config/test/wres/config/yaml/DeclarationValidatorTest.java +++ b/wres-config/test/wres/config/yaml/DeclarationValidatorTest.java @@ -80,6 +80,7 @@ import wres.statistics.generated.Threshold; import wres.statistics.generated.TimeScale; import wres.statistics.generated.GeometryTuple; +import wres.statistics.generated.TimeWindow; /** * Tests the {@link DeclarationValidator}. @@ -2631,6 +2632,31 @@ void testInconsistentGraphicsOrientationAndPoolingDeclarationProducesError() thr StatusLevel.ERROR ) ); } + @Test + void testCombinationOfExplicitAndGeneratedPoolsProducesWarning() + { + TimePools leadTimePools = new TimePools( java.time.Duration.ofHours( 1 ), + java.time.Duration.ofHours( 2 ) ); + + TimeWindow timeWindow = TimeWindow.newBuilder() + .build(); + Set timeWindows = Set.of( timeWindow ); + EvaluationDeclaration declaration = + EvaluationDeclarationBuilder.builder() + .left( this.defaultDataset ) + .right( this.defaultDataset ) + .leadTimePools( leadTimePools ) + .timeWindows( timeWindows ) + .build(); + + List events = DeclarationValidator.validate( declaration ); + + assertTrue( DeclarationValidatorTest.contains( events, + "the resulting pools from all sources will be added " + + "together", + StatusLevel.WARN ) ); + } + @Test void testInvalidDeclarationStringProducesSchemaValidationError() throws IOException // NOSONAR { diff --git a/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java b/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java index cb74a73a9c..e8faf7bd27 100644 --- a/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java +++ b/wres-io/src/wres/io/retrieving/database/TimeSeriesRetriever.java @@ -1578,7 +1578,8 @@ private Instant getOrInferUpperValidTime( TimeWindowOuter timeWindow ) Instant upperValidTime = Instant.MAX; // Upper bound present - if ( !timeWindow.getLatestValidTime().equals( Instant.MAX ) ) + if ( !timeWindow.getLatestValidTime() + .equals( Instant.MAX ) ) { upperValidTime = timeWindow.getLatestValidTime(); } @@ -1586,11 +1587,14 @@ private Instant getOrInferUpperValidTime( TimeWindowOuter timeWindow ) else { // Both the latest reference time and the latest lead duration available? - if ( !timeWindow.getLatestReferenceTime().equals( Instant.MAX ) - && !timeWindow.getLatestLeadDuration().equals( TimeWindowOuter.DURATION_MAX ) ) + if ( !timeWindow.getLatestReferenceTime() + .equals( Instant.MAX ) + && !timeWindow.getLatestLeadDuration() + .equals( TimeWindowOuter.DURATION_MAX ) ) { // Use the upper reference time plus upper lead duration - upperValidTime = timeWindow.getLatestReferenceTime().plus( timeWindow.getLatestLeadDuration() ); + upperValidTime = timeWindow.getLatestReferenceTime() + .plus( timeWindow.getLatestLeadDuration() ); } } @@ -1613,7 +1617,8 @@ private void addReferenceTimeBoundsToScript( DataScripter script, TimeWindowOute Objects.requireNonNull( filter ); // Lower and upper bounds are equal - if ( filter.getEarliestReferenceTime().equals( filter.getLatestReferenceTime() ) ) + if ( filter.getEarliestReferenceTime() + .equals( filter.getLatestReferenceTime() ) ) { OffsetDateTime referenceTime = OffsetDateTime.ofInstant( filter.getEarliestReferenceTime(), ZoneId.of( "UTC" ) ); @@ -1623,7 +1628,8 @@ private void addReferenceTimeBoundsToScript( DataScripter script, TimeWindowOute else { // Lower bound - if ( !filter.getEarliestReferenceTime().equals( Instant.MIN ) ) + if ( !filter.getEarliestReferenceTime() + .equals( Instant.MIN ) ) { OffsetDateTime lowerReferenceTime = OffsetDateTime.ofInstant( filter.getEarliestReferenceTime(), ZoneId.of( "UTC" ) ); @@ -1635,7 +1641,8 @@ private void addReferenceTimeBoundsToScript( DataScripter script, TimeWindowOute } // Upper bound - if ( !filter.getLatestReferenceTime().equals( Instant.MAX ) ) + if ( !filter.getLatestReferenceTime() + .equals( Instant.MAX ) ) { OffsetDateTime upperReferenceTime = OffsetDateTime.ofInstant( filter.getLatestReferenceTime(), ZoneId.of( "UTC" ) );