Merge pull request #561 from ie3-institute/sp/#267-icon-weather-schem…

…e-adaptions

Adapt column scheme of ICON weather data 2.0

.gitattributes

@@ -1,3 +1,4 @@
 # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line
 # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true'
 src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf
+src/test/resources/testContainerFiles/influxDb/icon/weather.txt eol=lf

CHANGELOG.md

@@ -42,6 +42,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
       - DWD COSMO model
         - `diffuseirradiation` to `diffuseirradiance`
         - `directirradiation` to `directirradiance`
+      - ICON model:
+        - `"datum"` to `"time"`
+	-   Force user to provide time stamp pattern to `CouchbaseWeatherSource` to ensure harmonized querying
 
 ## [2.1.0] - 2022-01-05
 

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java

@@ -29,7 +29,6 @@
  */
 public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory {
   /* Redefine the column names to meet the icon specifications */
-  private static final String TIME = "datum";
   private static final String DIFFUSE_IRRADIANCE = "aswdifdS";
   private static final String DIRECT_IRRADIANCE = "aswdirS";
   private static final String TEMPERATURE = "t2m";

src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java

@@ -25,7 +25,6 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import org.locationtech.jts.geom.Point;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -43,50 +42,120 @@ public class CouchbaseWeatherSource implements WeatherSource {
   private final CouchbaseConnector connector;
   private final IdCoordinateSource coordinateSource;
   private final String coordinateIdColumnName;
+  private final String timeStampPattern;
 
   /**
    * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
-   * connector. This convenient constructor uses the {@link
-   * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix.
+   * connector. This convenient constructor uses {@link CouchbaseWeatherSource#DEFAULT_KEY_PREFIX}
+   * as key prefix and {@link CouchbaseWeatherSource#DEFAULT_TIMESTAMP_PATTERN} as timestamp
+   * pattern.
    *
    * @param connector Connector, that establishes the connection to the couchbase instance
    * @param coordinateSource Source to obtain actual coordinates from
    * @param coordinateIdColumnName Name of the column containing the information about the
    *     coordinate identifier
    * @param weatherFactory Factory to transfer field to value mapping into actual java object
    *     instances
+   * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector,
+   *     IdCoordinateSource, String, TimeBasedWeatherValueFactory, String)} instead
    */
+  @Deprecated(since = "3.0", forRemoval = true)
   public CouchbaseWeatherSource(
       CouchbaseConnector connector,
       IdCoordinateSource coordinateSource,
       String coordinateIdColumnName,
       TimeBasedWeatherValueFactory weatherFactory) {
-    this(connector, coordinateSource, coordinateIdColumnName, DEFAULT_KEY_PREFIX, weatherFactory);
+    this(
+        connector,
+        coordinateSource,
+        coordinateIdColumnName,
+        weatherFactory,
+        DEFAULT_TIMESTAMP_PATTERN);
   }
 
   /**
    * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
-   * connector
+   * connector. This convenient constructor uses {@link CouchbaseWeatherSource#DEFAULT_KEY_PREFIX}
+   * as key prefix.
    *
    * @param connector Connector, that establishes the connection to the couchbase instance
    * @param coordinateSource Source to obtain actual coordinates from
    * @param coordinateIdColumnName Name of the column containing the information about the
    *     coordinate identifier
+   * @param weatherFactory Factory to transfer field to value mapping into actual java object
+   *     instances
+   * @param timeStampPattern Pattern of time stamps to parse
+   */
+  public CouchbaseWeatherSource(
+      CouchbaseConnector connector,
+      IdCoordinateSource coordinateSource,
+      String coordinateIdColumnName,
+      TimeBasedWeatherValueFactory weatherFactory,
+      String timeStampPattern) {
+    this(
+        connector,
+        coordinateSource,
+        coordinateIdColumnName,
+        DEFAULT_KEY_PREFIX,
+        weatherFactory,
+        timeStampPattern);
+  }
+
+  /**
+   * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
+   * connector. This convenient constructor uses {@link
+   * CouchbaseWeatherSource#DEFAULT_TIMESTAMP_PATTERN} as timestamp pattern.
+   *
+   * @param connector Connector, that establishes the connection to the couchbase instance
+   * @param coordinateSource Source to obtain actual coordinates from
    * @param keyPrefix Prefix of entries, that belong to weather
    * @param weatherFactory Factory to transfer field to value mapping into actual java object
    *     instances
+   * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector,
+   *     IdCoordinateSource, String, String, TimeBasedWeatherValueFactory, String)} instead
    */
+  @Deprecated(since = "3.0", forRemoval = true)
   public CouchbaseWeatherSource(
       CouchbaseConnector connector,
       IdCoordinateSource coordinateSource,
       String coordinateIdColumnName,
       String keyPrefix,
       TimeBasedWeatherValueFactory weatherFactory) {
+    this(
+        connector,
+        coordinateSource,
+        coordinateIdColumnName,
+        keyPrefix,
+        weatherFactory,
+        DEFAULT_TIMESTAMP_PATTERN);
+  }
+
+  /**
+   * Instantiate a weather source utilising a connection to a couchbase instance obtained via the
+   * connector
+   *
+   * @param connector Connector, that establishes the connection to the couchbase instance
+   * @param coordinateSource Source to obtain actual coordinates from
+   * @param coordinateIdColumnName Name of the column containing the information about the
+   *     coordinate identifier
+   * @param keyPrefix Prefix of entries, that belong to weather
+   * @param weatherFactory Factory to transfer field to value mapping into actual java object
+   *     instances
+   * @param timeStampPattern Pattern of time stamps to parse
+   */
+  public CouchbaseWeatherSource(
+      CouchbaseConnector connector,
+      IdCoordinateSource coordinateSource,
+      String coordinateIdColumnName,
+      String keyPrefix,
+      TimeBasedWeatherValueFactory weatherFactory,
+      String timeStampPattern) {
     this.connector = connector;
     this.coordinateSource = coordinateSource;
     this.coordinateIdColumnName = coordinateIdColumnName;
     this.keyPrefix = keyPrefix;
     this.weatherFactory = weatherFactory;
+    this.timeStampPattern = timeStampPattern;
   }
 
   @Override
@@ -118,7 +187,7 @@ public Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
           Set<TimeBasedValue<WeatherValue>> weatherInputs =
               jsonWeatherInputs.stream()
                   .map(this::toTimeBasedWeatherValue)
-                  .flatMap(o -> o.map(Stream::of).orElseGet(Stream::empty))
+                  .flatMap(Optional::stream)
                   .collect(Collectors.toSet());
           IndividualTimeSeries<WeatherValue> weatherTimeSeries =
               new IndividualTimeSeries<>(null, weatherInputs);
@@ -132,7 +201,7 @@ public Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
   @Override
   public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Point coordinate) {
     Optional<Integer> coordinateId = coordinateSource.getId(coordinate);
-    if (!coordinateId.isPresent()) {
+    if (coordinateId.isEmpty()) {
       logger.warn("Unable to match coordinate {} to a coordinate ID", coordinate);
       return Optional.empty();
     }
@@ -164,7 +233,7 @@ public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Poi
   public String generateWeatherKey(ZonedDateTime time, Integer coordinateId) {
     String key = keyPrefix + "::";
     key += coordinateId + "::";
-    key += time.format(DateTimeFormatter.ofPattern(DEFAULT_TIMESTAMP_PATTERN));
+    key += time.format(DateTimeFormatter.ofPattern(timeStampPattern));
     return key;
   }
 
@@ -198,7 +267,7 @@ private Optional<TimeBasedWeatherValueData> toTimeBasedWeatherValueData(JsonObje
     Integer coordinateId = jsonObj.getInt(coordinateIdColumnName);
     jsonObj.removeKey(coordinateIdColumnName);
     Optional<Point> coordinate = coordinateSource.getCoordinate(coordinateId);
-    if (!coordinate.isPresent()) {
+    if (coordinate.isEmpty()) {
       logger.warn("Unable to match coordinate ID {} to a coordinate", coordinateId);
       return Optional.empty();
     }
@@ -220,7 +289,7 @@ private Optional<TimeBasedWeatherValueData> toTimeBasedWeatherValueData(JsonObje
    */
   public Optional<TimeBasedValue<WeatherValue>> toTimeBasedWeatherValue(JsonObject jsonObj) {
     Optional<TimeBasedWeatherValueData> data = toTimeBasedWeatherValueData(jsonObj);
-    if (!data.isPresent()) {
+    if (data.isEmpty()) {
       logger.warn("Unable to parse json object");
       logger.debug("The following json could not be parsed:\n{}", jsonObj);
       return Optional.empty();

src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java

@@ -28,6 +28,7 @@
 public class InfluxDbWeatherSource implements WeatherSource {
   private static final String BASIC_QUERY_STRING = "Select * from weather";
   private static final String WHERE = " where ";
+  private static final String AND = " and ";
   private static final String MEASUREMENT_NAME_WEATHER = "weather";
   private static final String COORDINATE_ID_COLUMN_NAME = "coordinate_id";
   private static final int MILLI_TO_NANO_FACTOR = 1000000;
@@ -179,15 +180,15 @@ private String createQueryStringForCoordinateAndTimeInterval(
     return BASIC_QUERY_STRING
         + WHERE
         + createCoordinateConstraintString(coordinateId)
-        + " and "
+        + AND
         + createTimeConstraint(timeInterval);
   }
 
   private String createQueryStringForCoordinateAndTime(ZonedDateTime date, int coordinateId) {
     return BASIC_QUERY_STRING
         + WHERE
         + createCoordinateConstraintString(coordinateId)
-        + " and "
+        + AND
         + createTimeConstraint(date);
   }
 
@@ -196,14 +197,19 @@ private String createQueryStringForTimeInterval(ClosedInterval<ZonedDateTime> ti
   }
 
   private String createTimeConstraint(ClosedInterval<ZonedDateTime> timeInterval) {
-    return "time >= "
+    return weatherValueFactory.getTimeFieldString()
+        + " >= "
         + timeInterval.getLower().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR
-        + " and time <= "
+        + AND
+        + weatherValueFactory.getTimeFieldString()
+        + " <= "
         + timeInterval.getUpper().toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR;
   }
 
   private String createTimeConstraint(ZonedDateTime date) {
-    return "time=" + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR;
+    return weatherValueFactory.getTimeFieldString()
+        + "="
+        + date.toInstant().toEpochMilli() * MILLI_TO_NANO_FACTOR;
   }
 
   private String createCoordinateConstraintString(int coordinateId) {

src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy

@@ -76,7 +76,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification {
 		def coordinate = CosmoWeatherTestData.COORDINATE_67775
 
 		def parameter = [
-			"datum"        : "2019-08-01 01:00:00",
+			"time"        : "2019-08-01 01:00:00",
 			"albRad"      : "13.015240669",
 			"asobS"       : "3.555093673828124",
 			"aswdifdS"    : "1.8088226191406245",
@@ -98,7 +98,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification {
 			"w20m"        : "-0.0100060345167524",
 			"w216m"       : "-0.030348050471342078",
 			"w65m"        : "-0.01817112027569893",
-			"z0"           : "0.955323922526438",
+			"z0"          : "0.955323922526438",
 			"coordinateId": "67775",
 			"p131m"       : "",
 			"p20m"        : "",

src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy

@@ -31,7 +31,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour
 	BucketDefinition bucketDefinition = new BucketDefinition("ie3_in")
 
 	@Shared
-	CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:latest").withBucket(bucketDefinition)
+	CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition)
 	.withExposedPorts(8091, 8092, 8093, 8094, 11210)
 
 	@Shared
@@ -62,8 +62,9 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour
 				"--dataset", "file:///home/weather.json")
 
 		def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password)
-		def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx"))
-		source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory)
+		def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx"
+		def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern))
+		source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern)
 	}
 
 	def "The test container can establish a valid connection"() {