From be68e40b778bd5f13e24e5130c1f2847a6f2b516 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 6 Feb 2024 14:39:27 +0100 Subject: [PATCH 1/7] Refactor WeatherSource and WeatherSourceWrapper. --- CHANGELOG.md | 1 + .../ie3/simona/config/ConfigFailFast.scala | 130 +++++++- .../service/weather/WeatherService.scala | 3 +- .../service/weather/WeatherSource.scala | 283 +++++------------- .../weather/WeatherSourceWrapper.scala | 39 +-- .../simona/config/ConfigFailFastSpec.scala | 151 ++++++++-- .../service/weather/WeatherSourceSpec.scala | 35 --- 7 files changed, 343 insertions(+), 299 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ea6909c12..33c26705f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replaced akka with pekko [#641](https://github.com/ie3-institute/simona/issues/641) - Use `ThermalGrid` to calculate thermal environment of a heat pump [#315](https://github.com/ie3-institute/simona/issues/315) - Enable windows path as config parameters [#549](https://github.com/ie3-institute/simona/issues/549) +- Refactor `WeatherSource` and `WeatherSourceWrapper` [#180](https://github.com/ie3-institute/simona/issues/180) ### Fixed - Removed a repeated line in the documentation of vn_simona config [#658](https://github.com/ie3-institute/simona/issues/658) diff --git a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala index ab3b3895a9..0e7bfd6502 100644 --- a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala @@ -8,8 +8,15 @@ package edu.ie3.simona.config import com.typesafe.config.{Config, ConfigException} import com.typesafe.scalalogging.LazyLogging +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ + CouchbaseParams, + InfluxDb1xParams, + SampleParams, + SqlParams +} import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.InfluxDb1x import edu.ie3.simona.config.SimonaConfig.{ + BaseCsvParams, BaseOutputConfig, RefSystemConfig, ResultKafkaParams @@ -18,11 +25,14 @@ import edu.ie3.simona.exceptions.InvalidConfigParameterException import edu.ie3.simona.io.result.ResultSinkType import edu.ie3.simona.model.participant.load.{LoadModelBehaviour, LoadReference} import edu.ie3.simona.service.primary.PrimaryServiceProxy -import edu.ie3.simona.service.weather.WeatherSource +import edu.ie3.simona.service.weather.WeatherSource.WeatherScheme import edu.ie3.simona.util.CollectionUtils +import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ + checkCouchbaseParams, checkInfluxDb1xParams, - checkKafkaParams + checkKafkaParams, + checkSqlParams } import edu.ie3.simona.util.ConfigUtil.{CsvConfigUtil, NotifierIdentifier} import edu.ie3.util.scala.ReflectionTools @@ -527,8 +537,120 @@ case object ConfigFailFast extends LazyLogging { PrimaryServiceProxy.checkConfig(primary) private def checkWeatherDataSource( - dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource - ): Unit = WeatherSource.checkConfig(dataSourceConfig) + weatherDataSourceCfg: SimonaConfig.Simona.Input.Weather.Datasource + ): Unit = { + // check coordinate source + val definedCoordinateSource: String = checkCoordinateSource( + weatherDataSourceCfg.coordinateSource + ) + + /* Check, if the column scheme is supported */ + if (!WeatherScheme.isEligibleInput(weatherDataSourceCfg.scheme)) + throw new InvalidConfigParameterException( + s"The weather data scheme '${weatherDataSourceCfg.scheme}' is not supported. Supported schemes:\n\t${WeatherScheme.values + .mkString("\n\t")}" + ) + + // check weather source parameters + val supportedWeatherSources = + Set("influxdb1x", "csv", "sql", "couchbase", "sample") + val definedWeatherSources = Vector( + weatherDataSourceCfg.sampleParams, + weatherDataSourceCfg.csvParams, + weatherDataSourceCfg.influxDb1xParams, + weatherDataSourceCfg.couchbaseParams, + weatherDataSourceCfg.sqlParams + ).filter(_.isDefined) + + // check that only one source is defined + if (definedWeatherSources.size > 1) + throw new InvalidConfigParameterException( + s"Multiple weather sources defined: '${definedWeatherSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + + s"Please define only one source!\nAvailable sources:\n\t${supportedWeatherSources.mkString("\n\t")}" + ) + + definedWeatherSources.headOption.flatten match { + case Some(baseCsvParams: BaseCsvParams) => + checkBaseCsvParams(baseCsvParams, "WeatherSource") + case Some(params: CouchbaseParams) => + checkCouchbaseParams(params) + case Some(InfluxDb1xParams(database, _, url)) => + checkInfluxDb1xParams("WeatherSource", url, database) + case Some(params: SqlParams) => + checkSqlParams(params) + case Some(_: SampleParams) => + // sample weather, no check required + // coordinate source must be sample coordinate source + if (weatherDataSourceCfg.coordinateSource.sampleParams.isEmpty) { + // cannot use sample weather source with other combination of weather source than sample weather source + throw new InvalidConfigParameterException( + s"Invalid coordinate source " + + s"'$definedCoordinateSource' defined for SampleWeatherSource. " + + "Please adapt the configuration to use sample coordinate source for weather data!" + ) + } + case None | Some(_) => + throw new InvalidConfigParameterException( + s"No weather source defined! This is currently not supported! Please provide the config parameters for one " + + s"of the following weather sources:\n\t${supportedWeatherSources.mkString("\n\t")}" + ) + } + } + + /** Check the provided coordinate id data source configuration to ensure its + * validity. For any invalid configuration parameters exceptions are thrown. + * + * @param coordinateSourceConfig + * the config to be checked + * @return + * the name of the defined + * [[edu.ie3.datamodel.io.source.IdCoordinateSource]] + */ + private def checkCoordinateSource( + coordinateSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource + ): String = { + val supportedCoordinateSources = Set("csv", "sql", "sample") + val definedCoordSources = Vector( + coordinateSourceConfig.sampleParams, + coordinateSourceConfig.csvParams, + coordinateSourceConfig.sqlParams + ).filter(_.isDefined) + + // check that only one source is defined + if (definedCoordSources.size > 1) + throw new InvalidConfigParameterException( + s"Multiple coordinate sources defined: '${definedCoordSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + + s"Please define only one source!\nAvailable sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" + ) + + definedCoordSources.headOption.flatten match { + case Some(baseCsvParams: BaseCsvParams) => + checkBaseCsvParams(baseCsvParams, "CoordinateSource") + + // check the grid model configuration + val gridModel = coordinateSourceConfig.gridModel.toLowerCase + if (gridModel != "icon" && gridModel != "cosmo") { + throw new InvalidConfigParameterException( + s"Grid model '$gridModel' is not supported!" + ) + } + + "csv" + case Some(sqlParams: SqlParams) => + checkSqlParams(sqlParams) + "sql" + case Some( + _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams + ) => + "sample" + case None | Some(_) => + throw new InvalidConfigParameterException( + s"No coordinate source defined! This is currently not supported! Please provide the config parameters for one " + + s"of the following coordinate sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" + ) + } + + } /** Check the config sub tree for output parameterization * diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala index 12b2e6e2cb..0aae5e9675 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala @@ -123,8 +123,7 @@ final case class WeatherService( ): Try[(WeatherInitializedStateData, Option[Long])] = initServiceData match { case InitWeatherServiceStateData(sourceDefinition) => - val weatherSource = - WeatherSource(sourceDefinition, simulationStart) + val weatherSource = WeatherSource(sourceDefinition) /* What is the first tick to be triggered for? And what are further activation ticks */ val (maybeNextTick, furtherActivationTicks) = SortedDistinctSeq( diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index e054b01715..4f8c0127eb 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -6,6 +6,7 @@ package edu.ie3.simona.service.weather +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.SqlConnector import edu.ie3.datamodel.io.factory.timeseries.{ CosmoIdCoordinateFactory, @@ -30,17 +31,11 @@ import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, WeightedCoordinates } -import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ - checkCouchbaseParams, - checkInfluxDb1xParams, - checkSqlParams -} import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.scala.quantities.WattsPerSquareMeter -import org.locationtech.jts.geom.{Coordinate, Point} +import org.locationtech.jts.geom.{Coordinate, Point, Polygon} import squants.motion.MetersPerSecond import squants.thermal.Kelvin import tech.units.indriya.ComparableQuantity @@ -123,23 +118,13 @@ trait WeatherSource { case Some(exactHit) => /* The queried coordinate hit one of the weather coordinates. Don't average and take it directly */ Success(Vector(exactHit)) - case None if nearestCoords.size < amountOfInterpolationCoords => - Failure( - ServiceException( - s"There are not enough coordinates for averaging. Found ${nearestCoords.size} within the given distance of " + - s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." - ) - ) case None => - /* Check if enough coordinates are within the coordinate distance limit */ - val nearestCoordsInMaxDistance = nearestCoords.filter(coordDistance => - coordDistance.getDistance - .isLessThan(maxCoordinateDistance) - ) - if (nearestCoordsInMaxDistance.size < amountOfInterpolationCoords) { + /* There aren't enough coordinates inside the given distance */ + if (nearestCoords.size < amountOfInterpolationCoords) { Failure( ServiceException( - s"There are not enough coordinates within the max coordinate distance of $maxCoordinateDistance. Found ${nearestCoordsInMaxDistance.size} but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." + s"There are not enough coordinates for averaging. Found ${nearestCoords.size} within the given distance of " + + s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." ) ) } else { @@ -293,136 +278,48 @@ trait WeatherSource { object WeatherSource { def apply( - dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource, - simulationStart: ZonedDateTime - ): WeatherSource = - checkConfig(dataSourceConfig)(simulationStart) - - /** Check the provided weather data source configuration to ensure its - * validity. If the configuration is valid, a function to build the - * corresponding [[WeatherSource]] instance is returned. For any invalid - * configuration parameters exceptions are thrown. - * - * @param weatherDataSourceCfg - * the config to be checked - * @return - * a function that can be used to actually build the configured weather - * data source - */ - def checkConfig( weatherDataSourceCfg: SimonaConfig.Simona.Input.Weather.Datasource - ): ZonedDateTime => WeatherSource = { + )(implicit simulationStart: ZonedDateTime): WeatherSource = { + // get coordinate source + implicit val coordinateSourceFunction: IdCoordinateSource = + buildCoordinateSource(weatherDataSourceCfg.coordinateSource) - // check and get coordinate source - val coordinateSourceFunction: () => IdCoordinateSource = - checkCoordinateSource( - weatherDataSourceCfg.coordinateSource - ) - - /* Check, if the column scheme is supported */ - if (!WeatherScheme.isEligibleInput(weatherDataSourceCfg.scheme)) - throw new InvalidConfigParameterException( - s"The weather data scheme '${weatherDataSourceCfg.scheme}' is not supported. Supported schemes:\n\t${WeatherScheme.values - .mkString("\n\t")}" - ) - - // check weather source parameters - val supportedWeatherSources = - Set("influxdb1x", "csv", "sql", "couchbase", "sample") val definedWeatherSources = Vector( weatherDataSourceCfg.sampleParams, weatherDataSourceCfg.csvParams, weatherDataSourceCfg.influxDb1xParams, weatherDataSourceCfg.couchbaseParams, weatherDataSourceCfg.sqlParams - ).filter(_.isDefined) + ).find(_.isDefined).flatten - val timestampPattern: Option[String] = weatherDataSourceCfg.timestampPattern - val scheme: String = weatherDataSourceCfg.scheme - val resolution: Option[Long] = weatherDataSourceCfg.resolution - val distance: ComparableQuantity[Length] = + implicit val timestampPattern: Option[String] = + weatherDataSourceCfg.timestampPattern + implicit val scheme: String = weatherDataSourceCfg.scheme + implicit val resolution: Option[Long] = weatherDataSourceCfg.resolution + implicit val distance: ComparableQuantity[Length] = Quantities.getQuantity( weatherDataSourceCfg.maxCoordinateDistance, Units.METRE ) - // check that only one source is defined - if (definedWeatherSources.size > 1) - throw new InvalidConfigParameterException( - s"Multiple weather sources defined: '${definedWeatherSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + - s"Please define only one source!\nAvailable sources:\n\t${supportedWeatherSources.mkString("\n\t")}" - ) - definedWeatherSources.headOption match { - case Some( - Some(baseCsvParams @ BaseCsvParams(csvSep, directoryPath, _)) - ) => - checkBaseCsvParams(baseCsvParams, "WeatherSource") - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - csvSep, - Paths.get(directoryPath), - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance - )(simulationStart) - case Some(Some(params: CouchbaseParams)) => - checkCouchbaseParams(params) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance - )(simulationStart) - case Some(Some(params @ InfluxDb1xParams(database, _, url))) => - checkInfluxDb1xParams("WeatherSource", url, database) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance - )(simulationStart) - case Some(Some(params: SqlParams)) => - checkSqlParams(params) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance - )(simulationStart) - case Some(Some(_: SampleParams)) => - // sample weather, no check required - // coordinate source must be sample coordinate source - // calling the function here is not an issue as the sample coordinate source is already - // an object (= no overhead costs) - coordinateSourceFunction() match { - case _: SampleWeatherSource.SampleIdCoordinateSource.type => - // all fine - (simulationStart: ZonedDateTime) => - new SampleWeatherSource()(simulationStart) - case coordinateSource => - // cannot use sample weather source with other combination of weather source than sample weather source - throw new InvalidConfigParameterException( - s"Invalid coordinate source " + - s"'${coordinateSource.getClass.getSimpleName}' defined for SampleWeatherSource. " + - "Please adapt the configuration to use sample coordinate source for weather data!" - ) - } - case None | Some(_) => - throw new InvalidConfigParameterException( - s"No weather source defined! This is currently not supported! Please provide the config parameters for one " + - s"of the following weather sources:\n\t${supportedWeatherSources.mkString("\n\t")}" + definedWeatherSources match { + case Some(BaseCsvParams(csvSep, directoryPath, _)) => + WeatherSourceWrapper( + csvSep, + Paths.get(directoryPath) ) + case Some(params: CouchbaseParams) => + WeatherSourceWrapper(params) + case Some(params: InfluxDb1xParams) => + WeatherSourceWrapper(params) + case Some(params: SqlParams) => + WeatherSourceWrapper(params) + case Some(_: SampleParams) => + new SampleWeatherSource()(simulationStart) + case None => + throw new SourceException( + s"Expected a WeatherSource, but no source where defined in $weatherDataSourceCfg." + ); } } @@ -437,99 +334,57 @@ object WeatherSource { * a function that can be used to actually build the configured coordinate * id data source */ - private def checkCoordinateSource( + private def buildCoordinateSource( coordinateSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource - ): () => IdCoordinateSource = { - val supportedCoordinateSources = Set("csv", "sql", "sample") + ): IdCoordinateSource = { val definedCoordSources = Vector( coordinateSourceConfig.sampleParams, coordinateSourceConfig.csvParams, coordinateSourceConfig.sqlParams - ).filter(_.isDefined) + ).find(_.isDefined).flatten - // check that only one source is defined - if (definedCoordSources.size > 1) - throw new InvalidConfigParameterException( - s"Multiple coordinate sources defined: '${definedCoordSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + - s"Please define only one source!\nAvailable sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" - ) - - // check source parameters - definedCoordSources.headOption match { + definedCoordSources match { case Some( - Some(baseCsvParams @ BaseCsvParams(csvSep, directoryPath, _)) + BaseCsvParams(csvSep, directoryPath, _) ) => - checkBaseCsvParams(baseCsvParams, "CoordinateSource") - val idCoordinateFactory = checkCoordinateFactory( - coordinateSourceConfig.gridModel - ) - () => - new CsvIdCoordinateSource( - idCoordinateFactory, - new CsvDataSource( - csvSep, - Paths.get(directoryPath), - new FileNamingStrategy() - ) + val idCoordinateFactory = + coordinateSourceConfig.gridModel.toLowerCase match { + case "icon" => new IconIdCoordinateFactory() + case "cosmo" => new CosmoIdCoordinateFactory() + } + + new CsvIdCoordinateSource( + idCoordinateFactory, + new CsvDataSource( + csvSep, + Paths.get(directoryPath), + new FileNamingStrategy() ) + ) case Some( - Some( - sqlParams @ SqlParams( - jdbcUrl, - userName, - password, - schemaName, - tableName - ) + SqlParams( + jdbcUrl, + userName, + password, + schemaName, + tableName ) ) => - checkSqlParams(sqlParams) - - () => - new SqlIdCoordinateSource( - new SqlConnector(jdbcUrl, userName, password), - schemaName, - tableName, - new SqlIdCoordinateFactory() - ) + new SqlIdCoordinateSource( + new SqlConnector(jdbcUrl, userName, password), + schemaName, + tableName, + new SqlIdCoordinateFactory() + ) case Some( - Some( - _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams - ) + _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams ) => // sample coordinates, no check required - () => SampleWeatherSource.SampleIdCoordinateSource - case None | Some(_) => - throw new InvalidConfigParameterException( - s"No coordinate source defined! This is currently not supported! Please provide the config parameters for one " + - s"of the following coordinate sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" - ) - } - } - - /** Check the provided coordinate grid model configuration to ensure its - * validity. If the configuration is valid, the corresponding - * IdCoordinateSource is returned. For any invalid configuration parameters - * exceptions are thrown. - * - * @param gridModel - * the grid model string to be checked - * @return - * a function that can be used to actually build the id coordinate factory - * for the grid model - */ - private def checkCoordinateFactory( - gridModel: String - ): IdCoordinateFactory = { - if (gridModel.isEmpty) - throw new InvalidConfigParameterException("No grid model defined!") - gridModel.toLowerCase() match { - case "icon" => new IconIdCoordinateFactory() - case "cosmo" => new CosmoIdCoordinateFactory() - case _ => - throw new InvalidConfigParameterException( - s"Grid model '$gridModel' is not supported!" - ) + SampleWeatherSource.SampleIdCoordinateSource + case None => + throw new SourceException( + s"Expected an IdCoordinateSource, but no source where defined in $coordinateSourceConfig." + ); } } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 2fc48cd5f9..46bad45414 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -49,7 +49,6 @@ import tech.units.indriya.ComparableQuantity import java.nio.file.Path import java.time.ZonedDateTime import javax.measure.quantity.Length - import scala.jdk.CollectionConverters.{IterableHasAsJava, MapHasAsScala} import scala.jdk.OptionConverters.RichOptional import scala.util.{Failure, Success, Try} @@ -200,14 +199,15 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { def apply( csvSep: String, - directoryPath: Path, - idCoordinateSourceFunction: () => IdCoordinateSource, + directoryPath: Path + )(implicit + simulationStart: ZonedDateTime, + idCoordinateSource: IdCoordinateSource, timestampPattern: Option[String], scheme: String, resolution: Option[Long], maxCoordinateDistance: ComparableQuantity[Length] - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { - val idCoordinateSource = idCoordinateSourceFunction() + ): WeatherSourceWrapper = { val source = new CsvWeatherSource( csvSep, directoryPath, @@ -227,23 +227,24 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { } def apply( - couchbaseParams: CouchbaseParams, - idCoordinateSourceFunction: () => IdCoordinateSource, + couchbaseParams: CouchbaseParams + )(implicit + simulationStart: ZonedDateTime, + idCoordinateSource: IdCoordinateSource, timestampPattern: Option[String], scheme: String, resolution: Option[Long], maxCoordinateDistance: ComparableQuantity[Length] - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { + ): WeatherSourceWrapper = { val couchbaseConnector = new CouchbaseConnector( couchbaseParams.url, couchbaseParams.bucketName, couchbaseParams.userName, couchbaseParams.password ) - val idCoordinateSource = idCoordinateSourceFunction() val source = new CouchbaseWeatherSource( couchbaseConnector, - idCoordinateSourceFunction(), + idCoordinateSource, couchbaseParams.coordinateColumnName, couchbaseParams.keyPrefix, buildFactory(scheme, timestampPattern), @@ -261,16 +262,17 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { } def apply( - influxDbParams: InfluxDb1xParams, - idCoordinateSourceFunction: () => IdCoordinateSource, + influxDbParams: InfluxDb1xParams + )(implicit + simulationStart: ZonedDateTime, + idCoordinateSource: IdCoordinateSource, timestampPattern: Option[String], scheme: String, resolution: Option[Long], maxCoordinateDistance: ComparableQuantity[Length] - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { + ): WeatherSourceWrapper = { val influxDb1xConnector = new InfluxDbConnector(influxDbParams.url, influxDbParams.database) - val idCoordinateSource = idCoordinateSourceFunction() val source = new InfluxDbWeatherSource( influxDb1xConnector, idCoordinateSource, @@ -288,19 +290,20 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { } def apply( - sqlParams: SqlParams, - idCoordinateSourceFunction: () => IdCoordinateSource, + sqlParams: SqlParams + )(implicit + simulationStart: ZonedDateTime, + idCoordinateSource: IdCoordinateSource, timestampPattern: Option[String], scheme: String, resolution: Option[Long], maxCoordinateDistance: ComparableQuantity[Length] - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { + ): WeatherSourceWrapper = { val sqlConnector = new SqlConnector( sqlParams.jdbcUrl, sqlParams.userName, sqlParams.password ) - val idCoordinateSource = idCoordinateSourceFunction() val source = new SqlWeatherSource( sqlConnector, idCoordinateSource, diff --git a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala index 33f9914a0f..1680535cd6 100644 --- a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala +++ b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala @@ -7,7 +7,12 @@ package edu.ie3.simona.config import com.typesafe.config.ConfigFactory -import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource +import edu.ie3.datamodel.io.source.csv.CsvWeatherSource +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ + CoordinateSource, + SampleParams +} import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.{Csv, InfluxDb1x} import edu.ie3.simona.config.SimonaConfig.Simona.Powerflow.Newtonraphson @@ -18,8 +23,8 @@ import edu.ie3.simona.test.common.{ConfigTestData, UnitSpec} import edu.ie3.simona.util.ConfigUtil.{CsvConfigUtil, NotifierIdentifier} import edu.ie3.util.TimeUtil -import java.time.{Duration, ZonedDateTime} import java.time.temporal.ChronoUnit +import java.time.{Duration, ZonedDateTime} class ConfigFailFastSpec extends UnitSpec with ConfigTestData { "Validating the configs" when { @@ -909,44 +914,138 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { /* Checking of primary source configuration is delegated to the specific actor. Tests are placed there */ "Checking weather data sources" should { - val checkWeatherDataSource = PrivateMethod[Unit](Symbol("checkWeatherDataSource")) + val csv: BaseCsvParams = + BaseCsvParams(",", "input", isHierarchic = false) + val sample = new SampleParams(true) + + val weatherDataSource = Datasource( + CoordinateSource( + None, + "icon", + Some( + SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource + .SampleParams(true) + ), + None + ), + None, + None, + None, + 50000d, + Some(360L), + None, + "icon", + None, + Some("yyyy-MM-dd HH:mm") + ) + "detects invalid weather data scheme" in { - val weatherDataSource = - new SimonaConfig.Simona.Input.Weather.Datasource( - CoordinateSource( - None, - "icon", - Some( - SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource - .SampleParams(true) - ), - None - ), - None, - None, - None, - 50000d, - Some(360L), - Some( - SimonaConfig.Simona.Input.Weather.Datasource.SampleParams(true) - ), - "this won't work", - None, - Some("yyyy-MM-dd HH:mm") + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource( + weatherDataSource.copy(scheme = "this won't work") ) + }.getMessage shouldBe "The weather data scheme 'this won't work' is not supported. " + + "Supported schemes:\n\ticon\n\tcosmo" + } + + "detect missing source" in { intercept[InvalidConfigParameterException] { ConfigFailFast invokePrivate checkWeatherDataSource( weatherDataSource ) - }.getMessage shouldBe "The weather data scheme 'this won't work' is not supported. Supported schemes:\n\ticon\n\tcosmo" + }.getMessage should startWith( + "No weather source defined! This is currently not supported! Please provide the config parameters for " + + "one of the following weather sources:" + ) + } + + "detect too many sources" in { + val tooManySources = weatherDataSource.copy( + csvParams = Some(csv), + sampleParams = Some(sample) + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource(tooManySources) + }.getMessage should startWith("Multiple weather sources defined:") + } + + "detects sample source mismatch" in { + val csvCoordinateSource = new CoordinateSource( + csvParams = Some(csv), + gridModel = "icon", + sampleParams = None, + sqlParams = None + ) + + val sampleMismatch = weatherDataSource.copy( + coordinateSource = csvCoordinateSource, + sampleParams = Some(sample) + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource(sampleMismatch) + }.getMessage shouldBe "Invalid coordinate source 'csv' defined for SampleWeatherSource. Please adapt the configuration to use sample coordinate source for weather data!" } } } + "Checking coordinate sources" should { + val checkCoordinateSource = + PrivateMethod[Unit](Symbol("checkCoordinateSource")) + val csvParams: BaseCsvParams = BaseCsvParams( + ",", + "input", + isHierarchic = false + ) + val sampleParams = + new SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams( + true + ) + + val coordinateSource = new CoordinateSource( + csvParams = None, + gridModel = "icon", + sampleParams = None, + sqlParams = None + ) + + "detect missing source" in { + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(coordinateSource) + }.getMessage should startWith( + "No coordinate source defined! This is currently not supported! Please provide the config parameters for one of the following coordinate sources" + ) + } + + "detect too many sources" in { + val tooManySources = coordinateSource.copy( + csvParams = Some(csvParams), + sampleParams = Some(sampleParams) + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(tooManySources) + }.getMessage should startWith("Multiple coordinate sources defined:") + } + + "detect invalid grid model" in { + val invalidGridModel = coordinateSource.copy( + csvParams = Some(csvParams), + gridModel = "invalid" + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(invalidGridModel) + }.getMessage should startWith("Grid model 'invalid' is not supported!") + } + + } + "validating the typesafe config" when { "checking the availability of pekko logger parameterization" should { val checkPekkoLoggers = PrivateMethod[Unit](Symbol("checkPekkoLoggers")) diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 063e46d1f7..da2f1a2de4 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -299,41 +299,6 @@ class WeatherSourceSpec extends UnitSpec { ) } } - - "return correct coordinate factory" in { - val checkCoordinateFactory = - PrivateMethod[IdCoordinateFactory](Symbol("checkCoordinateFactory")) - - val cases = Table( - ("gridModel", "expectedClass", "failureMessage"), - ( - "", - classOf[InvalidConfigParameterException], - "No grid model defined!" - ), - ("icon", classOf[IconIdCoordinateFactory], ""), - ("cosmo", classOf[CosmoIdCoordinateFactory], ""), - ( - "else", - classOf[InvalidConfigParameterException], - "Grid model 'else' is not supported!" - ) - ) - - forAll(cases) { (gridModel, expectedClass, failureMessage) => - val actual = - Try(WeatherSource invokePrivate checkCoordinateFactory(gridModel)) - - actual match { - case Success(factory) => - factory.getClass shouldBe expectedClass - - case Failure(exception) => - exception.getClass shouldBe expectedClass - exception.getMessage shouldBe failureMessage - } - } - } } } From 9fd9a212416730eedc4bd326ab010af48a73d26a Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 8 Mar 2024 10:54:00 +0100 Subject: [PATCH 2/7] Some improvements. --- .../service/weather/WeatherSource.scala | 33 +-- .../weather/WeatherSourceWrapper.scala | 192 ++++++++---------- 2 files changed, 95 insertions(+), 130 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 4f8c0127eb..118ff91834 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -31,6 +31,7 @@ import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, WeightedCoordinates } +import edu.ie3.simona.service.weather.WeatherSourceWrapper.buildPSDMSource import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits @@ -292,9 +293,13 @@ object WeatherSource { weatherDataSourceCfg.sqlParams ).find(_.isDefined).flatten - implicit val timestampPattern: Option[String] = - weatherDataSourceCfg.timestampPattern - implicit val scheme: String = weatherDataSourceCfg.scheme + if (definedWeatherSources.isEmpty) { + // should not happen, due to the config fail fast check + throw new SourceException( + s"Expected a WeatherSource, but no source where defined in $weatherDataSourceCfg." + ) + } + implicit val resolution: Option[Long] = weatherDataSourceCfg.resolution implicit val distance: ComparableQuantity[Length] = Quantities.getQuantity( @@ -302,25 +307,9 @@ object WeatherSource { Units.METRE ) - definedWeatherSources match { - case Some(BaseCsvParams(csvSep, directoryPath, _)) => - WeatherSourceWrapper( - csvSep, - Paths.get(directoryPath) - ) - case Some(params: CouchbaseParams) => - WeatherSourceWrapper(params) - case Some(params: InfluxDb1xParams) => - WeatherSourceWrapper(params) - case Some(params: SqlParams) => - WeatherSourceWrapper(params) - case Some(_: SampleParams) => - new SampleWeatherSource()(simulationStart) - case None => - throw new SourceException( - s"Expected a WeatherSource, but no source where defined in $weatherDataSourceCfg." - ); - } + buildPSDMSource(weatherDataSourceCfg, definedWeatherSources) + .map(WeatherSourceWrapper.apply) + .getOrElse(new SampleWeatherSource()) } /** Check the provided coordinate id data source configuration to ensure its diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 46bad45414..6ae5f52022 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -25,6 +25,8 @@ import edu.ie3.datamodel.io.source.{ IdCoordinateSource, WeatherSource => PsdmWeatherSource } +import edu.ie3.simona.config.SimonaConfig +import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ CouchbaseParams, InfluxDb1xParams, @@ -46,7 +48,7 @@ import edu.ie3.util.DoubleUtils.ImplicitDouble import edu.ie3.util.interval.ClosedInterval import tech.units.indriya.ComparableQuantity -import java.nio.file.Path +import java.nio.file.Paths import java.time.ZonedDateTime import javax.measure.quantity.Length import scala.jdk.CollectionConverters.{IterableHasAsJava, MapHasAsScala} @@ -198,128 +200,102 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { private val DEFAULT_RESOLUTION = 3600L def apply( - csvSep: String, - directoryPath: Path + source: PsdmWeatherSource )(implicit simulationStart: ZonedDateTime, idCoordinateSource: IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length] + distance: ComparableQuantity[Length] ): WeatherSourceWrapper = { - val source = new CsvWeatherSource( - csvSep, - directoryPath, - new FileNamingStrategy(), - idCoordinateSource, - buildFactory(scheme, timestampPattern) - ) - logger.info( - "Successfully initiated CsvWeatherSource as source for WeatherSourceWrapper." - ) WeatherSourceWrapper( source, idCoordinateSource, resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance + distance ) } - def apply( - couchbaseParams: CouchbaseParams + private[weather] def buildPSDMSource( + cfgParams: SimonaConfig.Simona.Input.Weather.Datasource, + definedWeatherSources: Option[Serializable] )(implicit - simulationStart: ZonedDateTime, - idCoordinateSource: IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length] - ): WeatherSourceWrapper = { - val couchbaseConnector = new CouchbaseConnector( - couchbaseParams.url, - couchbaseParams.bucketName, - couchbaseParams.userName, - couchbaseParams.password - ) - val source = new CouchbaseWeatherSource( - couchbaseConnector, - idCoordinateSource, - couchbaseParams.coordinateColumnName, - couchbaseParams.keyPrefix, - buildFactory(scheme, timestampPattern), - "yyyy-MM-dd'T'HH:mm:ssxxx" - ) - logger.info( - "Successfully initiated CouchbaseWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance - ) - } + idCoordinateSource: IdCoordinateSource + ): Option[PsdmWeatherSource] = { + implicit val timestampPattern: Option[String] = + cfgParams.timestampPattern + implicit val scheme: String = cfgParams.scheme - def apply( - influxDbParams: InfluxDb1xParams - )(implicit - simulationStart: ZonedDateTime, - idCoordinateSource: IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length] - ): WeatherSourceWrapper = { - val influxDb1xConnector = - new InfluxDbConnector(influxDbParams.url, influxDbParams.database) - val source = new InfluxDbWeatherSource( - influxDb1xConnector, - idCoordinateSource, - buildFactory(scheme, timestampPattern) - ) - logger.info( - "Successfully initiated InfluxDbWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance - ) - } + val factory = buildFactory(scheme, timestampPattern) - def apply( - sqlParams: SqlParams - )(implicit - simulationStart: ZonedDateTime, - idCoordinateSource: IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length] - ): WeatherSourceWrapper = { - val sqlConnector = new SqlConnector( - sqlParams.jdbcUrl, - sqlParams.userName, - sqlParams.password - ) - val source = new SqlWeatherSource( - sqlConnector, - idCoordinateSource, - sqlParams.schemaName, - sqlParams.tableName, - buildFactory(scheme, timestampPattern) - ) - logger.info( - "Successfully initiated SqlWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance - ) + val source = definedWeatherSources.flatMap { + case BaseCsvParams(csvSep, directoryPath, _) => + // initializing a csv weather source + Some( + new CsvWeatherSource( + csvSep, + Paths.get(directoryPath), + new FileNamingStrategy(), + idCoordinateSource, + factory + ) + ) + case couchbaseParams: CouchbaseParams => + // initializing a couchbase weather source + val couchbaseConnector = new CouchbaseConnector( + couchbaseParams.url, + couchbaseParams.bucketName, + couchbaseParams.userName, + couchbaseParams.password + ) + Some( + new CouchbaseWeatherSource( + couchbaseConnector, + idCoordinateSource, + couchbaseParams.coordinateColumnName, + couchbaseParams.keyPrefix, + factory, + "yyyy-MM-dd'T'HH:mm:ssxxx" + ) + ) + case InfluxDb1xParams(database, _, url) => + // initializing an influxDb weather source + val influxDb1xConnector = + new InfluxDbConnector(url, database) + Some( + new InfluxDbWeatherSource( + influxDb1xConnector, + idCoordinateSource, + factory + ) + ) + case sqlParams: SqlParams => + // initializing a sql weather source + val sqlConnector = new SqlConnector( + sqlParams.jdbcUrl, + sqlParams.userName, + sqlParams.password + ) + Some( + new SqlWeatherSource( + sqlConnector, + idCoordinateSource, + sqlParams.schemaName, + sqlParams.tableName, + factory + ) + ) + case _ => + // no weather source is initialized + None + } + + source.foreach { source => + logger.info( + s"Successfully initialized ${source.getClass.getSimpleName} as source for WeatherSourceWrapper." + ) + } + + source } private def buildFactory(scheme: String, timestampPattern: Option[String]) = From d37031e2500bd06d3c69a26537756146fb80b62b Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 8 Mar 2024 11:10:40 +0100 Subject: [PATCH 3/7] fmt --- .../ie3/simona/config/ConfigFailFast.scala | 8 +++--- .../service/weather/WeatherSource.scala | 22 ++++----------- .../weather/WeatherSourceWrapper.scala | 28 +++++++++---------- .../simona/config/ConfigFailFastSpec.scala | 19 ++++++------- 4 files changed, 33 insertions(+), 44 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala index b2eb2eac34..8687556b90 100644 --- a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala @@ -12,7 +12,7 @@ import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ CouchbaseParams, InfluxDb1xParams, SampleParams, - SqlParams + SqlParams, } import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.InfluxDb1x import edu.ie3.simona.config.SimonaConfig.{ @@ -34,7 +34,7 @@ import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ checkCouchbaseParams, checkInfluxDb1xParams, checkKafkaParams, - checkSqlParams + checkSqlParams, } import edu.ie3.simona.util.ConfigUtil.{CsvConfigUtil, NotifierIdentifier} import edu.ie3.util.scala.ReflectionTools @@ -564,7 +564,7 @@ case object ConfigFailFast extends LazyLogging { weatherDataSourceCfg.csvParams, weatherDataSourceCfg.influxDb1xParams, weatherDataSourceCfg.couchbaseParams, - weatherDataSourceCfg.sqlParams + weatherDataSourceCfg.sqlParams, ).filter(_.isDefined) // check that only one source is defined @@ -618,7 +618,7 @@ case object ConfigFailFast extends LazyLogging { val definedCoordSources = Vector( coordinateSourceConfig.sampleParams, coordinateSourceConfig.csvParams, - coordinateSourceConfig.sqlParams + coordinateSourceConfig.sqlParams, ).filter(_.isDefined) // check that only one source is defined diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 86518854b6..cb8fec8db8 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -11,7 +11,6 @@ import edu.ie3.datamodel.io.connectors.SqlConnector import edu.ie3.datamodel.io.factory.timeseries.{ CosmoIdCoordinateFactory, IconIdCoordinateFactory, - IdCoordinateFactory, SqlIdCoordinateFactory, } import edu.ie3.datamodel.io.naming.FileNamingStrategy @@ -22,27 +21,18 @@ import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ -import edu.ie3.simona.exceptions.{ - InvalidConfigParameterException, - ServiceException, -} +import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, WeightedCoordinates, } -import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ - checkCouchbaseParams, - checkInfluxDb1xParams, - checkSqlParams, -} import edu.ie3.simona.service.weather.WeatherSourceWrapper.buildPSDMSource import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.scala.quantities.WattsPerSquareMeter -import org.locationtech.jts.geom.{Coordinate, Point, Polygon} +import org.locationtech.jts.geom.{Coordinate, Point} import squants.motion.MetersPerSecond import squants.thermal.Kelvin import tech.units.indriya.ComparableQuantity @@ -353,8 +343,8 @@ object WeatherSource { new CsvDataSource( csvSep, Paths.get(directoryPath), - new FileNamingStrategy() - ) + new FileNamingStrategy(), + ), ) case Some( SqlParams( @@ -362,14 +352,14 @@ object WeatherSource { userName, password, schemaName, - tableName + tableName, ) ) => new SqlIdCoordinateSource( new SqlConnector(jdbcUrl, userName, password), schemaName, tableName, - new SqlIdCoordinateFactory() + new SqlIdCoordinateFactory(), ) case Some( _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 16f7a4b84e..6ccc0942bd 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -74,7 +74,7 @@ private[weather] final case class WeatherSourceWrapper private ( source: PsdmWeatherSource, override val idCoordinateSource: IdCoordinateSource, resolution: Long, - maxCoordinateDistance: ComparableQuantity[Length] + maxCoordinateDistance: ComparableQuantity[Length], )( private implicit val simulationStart: ZonedDateTime ) extends SimonaWeatherSource @@ -92,7 +92,7 @@ private[weather] final case class WeatherSourceWrapper private ( */ override def getWeather( tick: Long, - weightedCoordinates: WeatherSource.WeightedCoordinates + weightedCoordinates: WeatherSource.WeightedCoordinates, ): WeatherMessage.WeatherData = { val dateTime = tick.toDateTime val interval = new ClosedInterval(dateTime, dateTime) @@ -100,7 +100,7 @@ private[weather] final case class WeatherSourceWrapper private ( val results = source .getWeather( interval, - coordinates + coordinates, ) .asScala .toMap @@ -126,7 +126,7 @@ private[weather] final case class WeatherSourceWrapper private ( point, { logger.warn(s"Received an unexpected point: $point") 0d - } + }, ) /* Sum up weight and contributions */ @@ -171,8 +171,8 @@ private[weather] final case class WeatherSourceWrapper private ( diffIrrWeight, dirIrrWeight, tempWeight, - windVelWeight - ) + windVelWeight, + ), ) } match { case (weatherData: WeatherData, weightSum: WeightSum) => @@ -206,7 +206,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { simulationStart: ZonedDateTime, idCoordinateSource: IdCoordinateSource, resolution: Option[Long], - distance: ComparableQuantity[Length] + distance: ComparableQuantity[Length], ): WeatherSourceWrapper = { WeatherSourceWrapper( source, @@ -218,7 +218,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { private[weather] def buildPSDMSource( cfgParams: SimonaConfig.Simona.Input.Weather.Datasource, - definedWeatherSources: Option[Serializable] + definedWeatherSources: Option[Serializable], )(implicit idCoordinateSource: IdCoordinateSource ): Option[PsdmWeatherSource] = { @@ -237,7 +237,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { Paths.get(directoryPath), new FileNamingStrategy(), idCoordinateSource, - factory + factory, ) ) case couchbaseParams: CouchbaseParams => @@ -246,7 +246,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { couchbaseParams.url, couchbaseParams.bucketName, couchbaseParams.userName, - couchbaseParams.password + couchbaseParams.password, ) Some( new CouchbaseWeatherSource( @@ -255,7 +255,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { couchbaseParams.coordinateColumnName, couchbaseParams.keyPrefix, factory, - "yyyy-MM-dd'T'HH:mm:ssxxx" + "yyyy-MM-dd'T'HH:mm:ssxxx", ) ) case InfluxDb1xParams(database, _, url) => @@ -266,7 +266,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { new InfluxDbWeatherSource( influxDb1xConnector, idCoordinateSource, - factory + factory, ) ) case sqlParams: SqlParams => @@ -274,7 +274,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { val sqlConnector = new SqlConnector( sqlParams.jdbcUrl, sqlParams.userName, - sqlParams.password + sqlParams.password, ) Some( new SqlWeatherSource( @@ -282,7 +282,7 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { idCoordinateSource, sqlParams.schemaName, sqlParams.tableName, - factory + factory, ) ) case _ => diff --git a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala index bb6a1fbc16..01d137019e 100644 --- a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala +++ b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala @@ -7,11 +7,10 @@ package edu.ie3.simona.config import com.typesafe.config.ConfigFactory -import edu.ie3.datamodel.io.source.csv.CsvWeatherSource import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ CoordinateSource, - SampleParams + SampleParams, } import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.{Csv, InfluxDb1x} @@ -939,7 +938,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource .SampleParams(true) ), - None + None, ), None, None, @@ -975,7 +974,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { "detect too many sources" in { val tooManySources = weatherDataSource.copy( csvParams = Some(csv), - sampleParams = Some(sample) + sampleParams = Some(sample), ) intercept[InvalidConfigParameterException] { @@ -988,12 +987,12 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { csvParams = Some(csv), gridModel = "icon", sampleParams = None, - sqlParams = None + sqlParams = None, ) val sampleMismatch = weatherDataSource.copy( coordinateSource = csvCoordinateSource, - sampleParams = Some(sample) + sampleParams = Some(sample), ) intercept[InvalidConfigParameterException] { @@ -1081,7 +1080,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { val csvParams: BaseCsvParams = BaseCsvParams( ",", "input", - isHierarchic = false + isHierarchic = false, ) val sampleParams = new SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams( @@ -1092,7 +1091,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { csvParams = None, gridModel = "icon", sampleParams = None, - sqlParams = None + sqlParams = None, ) "detect missing source" in { @@ -1106,7 +1105,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { "detect too many sources" in { val tooManySources = coordinateSource.copy( csvParams = Some(csvParams), - sampleParams = Some(sampleParams) + sampleParams = Some(sampleParams), ) intercept[InvalidConfigParameterException] { @@ -1117,7 +1116,7 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { "detect invalid grid model" in { val invalidGridModel = coordinateSource.copy( csvParams = Some(csvParams), - gridModel = "invalid" + gridModel = "invalid", ) intercept[InvalidConfigParameterException] { From 963d8efc2bee5553c6eb44b6284a4f18f3e2bfb3 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 7 May 2024 10:18:20 +0200 Subject: [PATCH 4/7] Adapting to newly introduced method in `IdCoordinateSource`. --- .../service/weather/SampleWeatherSource.scala | 9 +++ .../service/weather/WeatherSource.scala | 65 ++++++------------- .../service/weather/WeatherSourceSpec.scala | 41 +++++------- 3 files changed, 43 insertions(+), 72 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala index 5788aca327..2ecf881c3f 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala @@ -168,6 +168,15 @@ object SampleWeatherSource { else Vector.empty[CoordinateDistance].asJava } + + override def findCornerPoints( + coordinate: Point, + distance: ComparableQuantity[Length], + ): util.List[CoordinateDistance] = + findCornerPoints( + coordinate, + getClosestCoordinates(coordinate, 9, distance), + ) } // these lists contain the hourly weather values for each first of the month of 2011 + january of diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index cb8fec8db8..1872fca6c8 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -100,54 +100,27 @@ trait WeatherSource { ): Try[Iterable[CoordinateDistance]] = { val queryPoint = coordinate.toPoint - /* Go and get the nearest coordinates, that are known to the weather source */ - val nearestCoords = idCoordinateSource - .getClosestCoordinates( - queryPoint, - amountOfInterpolationCoords, - maxCoordinateDistance, - ) - .asScala + /* Go and get the corner coordinates, that are within a given distance */ + val possibleCornerPoints = idCoordinateSource.findCornerPoints( + queryPoint, + maxCoordinateDistance, + ) - nearestCoords.find(coordinateDistance => - coordinateDistance.getCoordinateB.equalsExact(queryPoint, 1e-6) - ) match { - case Some(exactHit) => - /* The queried coordinate hit one of the weather coordinates. Don't average and take it directly */ - Success(Vector(exactHit)) - case None => - /* There aren't enough coordinates inside the given distance */ - if (nearestCoords.size < amountOfInterpolationCoords) { - Failure( - ServiceException( - s"There are not enough coordinates for averaging. Found ${nearestCoords.size} within the given distance of " + - s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." - ) - ) - } else { - /* Check, if the queried coordinate is surrounded at each quadrant */ - val (topLeft, topRight, bottomLeft, bottomRight) = nearestCoords - .map(_.getCoordinateB) - .foldLeft((false, false, false, false)) { - case ((tl, tr, bl, br), point) => - ( - tl || (point.getX < queryPoint.getX && point.getY > queryPoint.getY), - tr || (point.getX > queryPoint.getX && point.getY > queryPoint.getY), - bl || (point.getX < queryPoint.getX && point.getY < queryPoint.getY), - br || (point.getX > queryPoint.getX && point.getY < queryPoint.getY), - ) - } + val nr = possibleCornerPoints.size() - /* There has to be a coordinate in each quadrant */ - if (topLeft && topRight && bottomLeft && bottomRight) - Success(nearestCoords) - else - Failure( - ServiceException( - s"The queried point shall be surrounded by $amountOfInterpolationCoords weather coordinates, which are in each quadrant. This is not the case." - ) - ) - } + if (nr == 1) { + // found one exact match + Success(possibleCornerPoints.asScala) + } else if (nr == amountOfInterpolationCoords) { + // found enough points for interpolating + Success(possibleCornerPoints.asScala) + } else { + Failure( + ServiceException( + s"There are not enough coordinates for averaging. Found ${possibleCornerPoints.size()} within the given distance of " + + s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." + ) + ) } } diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index a9daaa0f35..3da4ac7f85 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -6,16 +6,8 @@ package edu.ie3.simona.service.weather -import edu.ie3.datamodel.io.factory.timeseries.{ - CosmoIdCoordinateFactory, - IconIdCoordinateFactory, - IdCoordinateFactory, -} import edu.ie3.datamodel.io.source.IdCoordinateSource -import edu.ie3.simona.exceptions.{ - InvalidConfigParameterException, - ServiceException, -} +import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, @@ -35,7 +27,7 @@ import java.util.Optional import javax.measure.quantity.Length import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} class WeatherSourceSpec extends UnitSpec { private val coordinate0 = GeoUtils.buildPoint(51.47, 7.41) @@ -47,29 +39,17 @@ class WeatherSourceSpec extends UnitSpec { 9, ) match { case Failure(exception: ServiceException) => - exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." + exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." case _ => fail("You shall not pass!") } } "issue a ServiceException, if there are not enough coordinates in max distance available" in { DummyWeatherSource.getNearestCoordinatesWithDistances( AgentCoordinates(coordinate0.getY, coordinate0.getX), - 9, - ) match { - case Failure(exception: ServiceException) => - exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." - case _ => fail("You shall not pass!") - } - } - - "issue a ServiceException, if the queried coordinate is not surrounded by the found weather coordinates" in { - val agentCoordinates = AgentCoordinates(51.3, 7.3) - DummyWeatherSource.getNearestCoordinatesWithDistances( - agentCoordinates, - 4, + 5, ) match { case Failure(exception: ServiceException) => - exception.getMessage shouldBe "The queried point shall be surrounded by 4 weather coordinates, which are in each quadrant. This is not the case." + exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 5. Please make sure that there are enough coordinates within the given distance." case _ => fail("You shall not pass!") } } @@ -244,7 +224,7 @@ class WeatherSourceSpec extends UnitSpec { case Failure(exception: ServiceException) => exception.getMessage shouldBe "Determination of coordinate weights failed." exception.getCause shouldBe ServiceException( - "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." + "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." ) case _ => fail("You shall not pass!") } @@ -415,5 +395,14 @@ case object WeatherSourceSpec { ): util.List[CoordinateDistance] = { calculateCoordinateDistances(coordinate, n, coordinateToId.keySet.asJava) } + + override def findCornerPoints( + coordinate: Point, + distance: ComparableQuantity[Length], + ): util.List[CoordinateDistance] = + findCornerPoints( + coordinate, + getClosestCoordinates(coordinate, 9, distance), + ) } } From e4095289a4b29755684246e3379fb8ecc69a7151 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 24 Jun 2024 13:17:15 +0200 Subject: [PATCH 5/7] Adapting some parts. --- build.gradle | 2 +- src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala | 8 -------- .../ie3/simona/service/weather/SampleWeatherSource.scala | 4 ++++ .../ie3/simona/service/weather/WeatherSourceSpec.scala | 4 ++++ .../simona/service/weather/WeatherSourceWrapperSpec.scala | 6 ++---- .../ie3/simona/test/common/input/EmInputTestData.scala | 4 ---- 6 files changed, 11 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index c18367c433..192cee24d6 100644 --- a/build.gradle +++ b/build.gradle @@ -75,7 +75,7 @@ dependencies { /* Exclude our own nested dependencies */ exclude group: 'com.github.ie3-institute' } - implementation('com.github.ie3-institute:PowerSystemDataModel:5.0.1') { + implementation('com.github.ie3-institute:PowerSystemDataModel:6.0-SNAPSHOT') { exclude group: 'org.apache.logging.log4j' exclude group: 'org.slf4j' /* Exclude our own nested dependencies */ diff --git a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala index 782a4a9bb5..6373f495cd 100644 --- a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala @@ -15,14 +15,6 @@ import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ SqlParams, } import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.InfluxDb1x -import edu.ie3.simona.config.SimonaConfig.{ - BaseCsvParams, - BaseOutputConfig, - RefSystemConfig, - ResultKafkaParams, - Simona, - TransformerControlGroup, -} import edu.ie3.simona.config.SimonaConfig._ import edu.ie3.simona.exceptions.InvalidConfigParameterException import edu.ie3.simona.io.result.ResultSinkType diff --git a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala index 2ecf881c3f..c8045ced59 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala @@ -177,6 +177,10 @@ object SampleWeatherSource { coordinate, getClosestCoordinates(coordinate, 9, distance), ) + + override def validate(): Unit = { + /* nothing to do here */ + } } // these lists contain the hourly weather values for each first of the month of 2011 + january of diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index 3da4ac7f85..3aacd91ff0 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -404,5 +404,9 @@ case object WeatherSourceSpec { coordinate, getClosestCoordinates(coordinate, 9, distance), ) + + override def validate(): Unit = { + /* nothing to do here */ + } } } diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala index ab00e32bdf..3c937e0976 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala @@ -318,11 +318,9 @@ object WeatherSourceWrapperSpec { ), ) - override def getSourceFields[C <: WeatherValue]( - entityClass: Class[C] - ): Optional[util.Set[String]] = + override def getSourceFields: Optional[util.Set[String]] = // only required for validation - Optional.empty + Optional.empty() override def getWeather( timeInterval: ClosedInterval[ZonedDateTime] diff --git a/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala b/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala index 4c349c603e..893fcfd8dc 100644 --- a/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala +++ b/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala @@ -32,7 +32,6 @@ import edu.ie3.simona.util.ConfigUtil import edu.ie3.util.quantities.PowerSystemUnits._ import squants.energy.Kilowatts import tech.units.indriya.quantity.Quantities -import tech.units.indriya.unit.Units._ import java.util.UUID import scala.jdk.CollectionConverters.SeqHasAsJava @@ -78,9 +77,6 @@ trait EmInputTestData Quantities.getQuantity(5d, KILOWATT), Quantities.getQuantity(0.03, PU_PER_HOUR), Quantities.getQuantity(0.95, PU), - Quantities.getQuantity(20d, PERCENT), - Quantities.getQuantity(50000d, HOUR), - 100000, ) protected val householdStorageInput = new StorageInput( From dbe27c05e2e48eda6db3a747305b7be661efad6d Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 24 Jun 2024 13:25:51 +0200 Subject: [PATCH 6/7] Fixing `Codacy` issue. --- .../service/weather/WeatherSource.scala | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index 1872fca6c8..12a26c3f6c 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -106,21 +106,20 @@ trait WeatherSource { maxCoordinateDistance, ) - val nr = possibleCornerPoints.size() - - if (nr == 1) { - // found one exact match - Success(possibleCornerPoints.asScala) - } else if (nr == amountOfInterpolationCoords) { - // found enough points for interpolating - Success(possibleCornerPoints.asScala) - } else { - Failure( - ServiceException( - s"There are not enough coordinates for averaging. Found ${possibleCornerPoints.size()} within the given distance of " + - s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." + possibleCornerPoints.size() match { + case 1 => + // found one exact match + Success(possibleCornerPoints.asScala) + case nr if nr == amountOfInterpolationCoords => + // found enough points for interpolating + Success(possibleCornerPoints.asScala) + case invalidNo => + Failure( + ServiceException( + s"There are not enough coordinates for averaging. Found $invalidNo within the given distance of " + + s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." + ) ) - ) } } From b54294be67e0738acb0422cc200f4e7ecbda65b5 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 24 Jun 2024 13:51:56 +0200 Subject: [PATCH 7/7] Updating PSDM. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 192cee24d6..68ca2f3287 100644 --- a/build.gradle +++ b/build.gradle @@ -75,7 +75,7 @@ dependencies { /* Exclude our own nested dependencies */ exclude group: 'com.github.ie3-institute' } - implementation('com.github.ie3-institute:PowerSystemDataModel:6.0-SNAPSHOT') { + implementation('com.github.ie3-institute:PowerSystemDataModel:5.1.0') { exclude group: 'org.apache.logging.log4j' exclude group: 'org.slf4j' /* Exclude our own nested dependencies */