diff --git a/CHANGELOG.md b/CHANGELOG.md index c298ef6604..5371ca3012 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Rewrote PVModelTest from groovy to scala [#646](https://github.com/ie3-institute/simona/issues/646) - Making configuration of `RefSystem` via config optional [#769](https://github.com/ie3-institute/simona/issues/769) - Updated PSDM to version 5.1.0 [#835](https://github.com/ie3-institute/simona/issues/835) +- Refactor `WeatherSource` and `WeatherSourceWrapper` [#180](https://github.com/ie3-institute/simona/issues/180) ### Fixed - Removed a repeated line in the documentation of vn_simona config [#658](https://github.com/ie3-institute/simona/issues/658) diff --git a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala index 9b31c6d739..3437944c78 100644 --- a/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/simona/config/ConfigFailFast.scala @@ -8,17 +8,26 @@ package edu.ie3.simona.config import com.typesafe.config.{Config, ConfigException} import com.typesafe.scalalogging.LazyLogging +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ + CouchbaseParams, + InfluxDb1xParams, + SampleParams, + SqlParams, +} import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.InfluxDb1x import edu.ie3.simona.config.SimonaConfig._ import edu.ie3.simona.exceptions.InvalidConfigParameterException import edu.ie3.simona.io.result.ResultSinkType import edu.ie3.simona.model.participant.load.{LoadModelBehaviour, LoadReference} import edu.ie3.simona.service.primary.PrimaryServiceProxy -import edu.ie3.simona.service.weather.WeatherSource +import edu.ie3.simona.service.weather.WeatherSource.WeatherScheme import edu.ie3.simona.util.CollectionUtils +import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ + checkCouchbaseParams, checkInfluxDb1xParams, checkKafkaParams, + checkSqlParams, } import edu.ie3.simona.util.ConfigUtil.{CsvConfigUtil, NotifierIdentifier} import edu.ie3.util.scala.ReflectionTools @@ -539,8 +548,120 @@ case object ConfigFailFast extends LazyLogging { PrimaryServiceProxy.checkConfig(primary) private def checkWeatherDataSource( - dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource - ): Unit = WeatherSource.checkConfig(dataSourceConfig) + weatherDataSourceCfg: SimonaConfig.Simona.Input.Weather.Datasource + ): Unit = { + // check coordinate source + val definedCoordinateSource: String = checkCoordinateSource( + weatherDataSourceCfg.coordinateSource + ) + + /* Check, if the column scheme is supported */ + if (!WeatherScheme.isEligibleInput(weatherDataSourceCfg.scheme)) + throw new InvalidConfigParameterException( + s"The weather data scheme '${weatherDataSourceCfg.scheme}' is not supported. Supported schemes:\n\t${WeatherScheme.values + .mkString("\n\t")}" + ) + + // check weather source parameters + val supportedWeatherSources = + Set("influxdb1x", "csv", "sql", "couchbase", "sample") + val definedWeatherSources = Vector( + weatherDataSourceCfg.sampleParams, + weatherDataSourceCfg.csvParams, + weatherDataSourceCfg.influxDb1xParams, + weatherDataSourceCfg.couchbaseParams, + weatherDataSourceCfg.sqlParams, + ).filter(_.isDefined) + + // check that only one source is defined + if (definedWeatherSources.size > 1) + throw new InvalidConfigParameterException( + s"Multiple weather sources defined: '${definedWeatherSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + + s"Please define only one source!\nAvailable sources:\n\t${supportedWeatherSources.mkString("\n\t")}" + ) + + definedWeatherSources.headOption.flatten match { + case Some(baseCsvParams: BaseCsvParams) => + checkBaseCsvParams(baseCsvParams, "WeatherSource") + case Some(params: CouchbaseParams) => + checkCouchbaseParams(params) + case Some(InfluxDb1xParams(database, _, url)) => + checkInfluxDb1xParams("WeatherSource", url, database) + case Some(params: SqlParams) => + checkSqlParams(params) + case Some(_: SampleParams) => + // sample weather, no check required + // coordinate source must be sample coordinate source + if (weatherDataSourceCfg.coordinateSource.sampleParams.isEmpty) { + // cannot use sample weather source with other combination of weather source than sample weather source + throw new InvalidConfigParameterException( + s"Invalid coordinate source " + + s"'$definedCoordinateSource' defined for SampleWeatherSource. " + + "Please adapt the configuration to use sample coordinate source for weather data!" + ) + } + case None | Some(_) => + throw new InvalidConfigParameterException( + s"No weather source defined! This is currently not supported! Please provide the config parameters for one " + + s"of the following weather sources:\n\t${supportedWeatherSources.mkString("\n\t")}" + ) + } + } + + /** Check the provided coordinate id data source configuration to ensure its + * validity. For any invalid configuration parameters exceptions are thrown. + * + * @param coordinateSourceConfig + * the config to be checked + * @return + * the name of the defined + * [[edu.ie3.datamodel.io.source.IdCoordinateSource]] + */ + private def checkCoordinateSource( + coordinateSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource + ): String = { + val supportedCoordinateSources = Set("csv", "sql", "sample") + val definedCoordSources = Vector( + coordinateSourceConfig.sampleParams, + coordinateSourceConfig.csvParams, + coordinateSourceConfig.sqlParams, + ).filter(_.isDefined) + + // check that only one source is defined + if (definedCoordSources.size > 1) + throw new InvalidConfigParameterException( + s"Multiple coordinate sources defined: '${definedCoordSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + + s"Please define only one source!\nAvailable sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" + ) + + definedCoordSources.headOption.flatten match { + case Some(baseCsvParams: BaseCsvParams) => + checkBaseCsvParams(baseCsvParams, "CoordinateSource") + + // check the grid model configuration + val gridModel = coordinateSourceConfig.gridModel.toLowerCase + if (gridModel != "icon" && gridModel != "cosmo") { + throw new InvalidConfigParameterException( + s"Grid model '$gridModel' is not supported!" + ) + } + + "csv" + case Some(sqlParams: SqlParams) => + checkSqlParams(sqlParams) + "sql" + case Some( + _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams + ) => + "sample" + case None | Some(_) => + throw new InvalidConfigParameterException( + s"No coordinate source defined! This is currently not supported! Please provide the config parameters for one " + + s"of the following coordinate sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" + ) + } + + } /** Check the config sub tree for output parameterization * diff --git a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala index 37a5cd8fa9..c8045ced59 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/SampleWeatherSource.scala @@ -170,12 +170,13 @@ object SampleWeatherSource { } override def findCornerPoints( - point: Point, + coordinate: Point, distance: ComparableQuantity[Length], - ): util.List[CoordinateDistance] = { - // just a dummy implementation, because this is just a sample weather source - getClosestCoordinates(point, 4, distance) - } + ): util.List[CoordinateDistance] = + findCornerPoints( + coordinate, + getClosestCoordinates(coordinate, 9, distance), + ) override def validate(): Unit = { /* nothing to do here */ diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala index 7543dc74c8..a75b0c38fd 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherService.scala @@ -123,8 +123,7 @@ final case class WeatherService( ): Try[(WeatherInitializedStateData, Option[Long])] = initServiceData match { case InitWeatherServiceStateData(sourceDefinition) => - val weatherSource = - WeatherSource(sourceDefinition, simulationStart) + val weatherSource = WeatherSource(sourceDefinition) /* What is the first tick to be triggered for? And what are further activation ticks */ val (maybeNextTick, furtherActivationTicks) = SortedDistinctSeq( diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala index b996d5a475..12a26c3f6c 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSource.scala @@ -6,11 +6,11 @@ package edu.ie3.simona.service.weather +import edu.ie3.datamodel.exceptions.SourceException import edu.ie3.datamodel.io.connectors.SqlConnector import edu.ie3.datamodel.io.factory.timeseries.{ CosmoIdCoordinateFactory, IconIdCoordinateFactory, - IdCoordinateFactory, SqlIdCoordinateFactory, } import edu.ie3.datamodel.io.naming.FileNamingStrategy @@ -21,21 +21,13 @@ import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.simona.config.SimonaConfig import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource._ -import edu.ie3.simona.exceptions.{ - InvalidConfigParameterException, - ServiceException, -} +import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage.WeatherData import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, WeightedCoordinates, } -import edu.ie3.simona.util.ConfigUtil.CsvConfigUtil.checkBaseCsvParams -import edu.ie3.simona.util.ConfigUtil.DatabaseConfigUtil.{ - checkCouchbaseParams, - checkInfluxDb1xParams, - checkSqlParams, -} +import edu.ie3.simona.service.weather.WeatherSourceWrapper.buildPSDMSource import edu.ie3.simona.util.ParsableEnumeration import edu.ie3.util.geo.{CoordinateDistance, GeoUtils} import edu.ie3.util.quantities.PowerSystemUnits @@ -108,64 +100,26 @@ trait WeatherSource { ): Try[Iterable[CoordinateDistance]] = { val queryPoint = coordinate.toPoint - /* Go and get the nearest coordinates, that are known to the weather source */ - val nearestCoords = idCoordinateSource - .getClosestCoordinates( - queryPoint, - amountOfInterpolationCoords, - maxCoordinateDistance, - ) - .asScala + /* Go and get the corner coordinates, that are within a given distance */ + val possibleCornerPoints = idCoordinateSource.findCornerPoints( + queryPoint, + maxCoordinateDistance, + ) - nearestCoords.find(coordinateDistance => - coordinateDistance.getCoordinateB.equalsExact(queryPoint, 1e-6) - ) match { - case Some(exactHit) => - /* The queried coordinate hit one of the weather coordinates. Don't average and take it directly */ - Success(Vector(exactHit)) - case None if nearestCoords.size < amountOfInterpolationCoords => + possibleCornerPoints.size() match { + case 1 => + // found one exact match + Success(possibleCornerPoints.asScala) + case nr if nr == amountOfInterpolationCoords => + // found enough points for interpolating + Success(possibleCornerPoints.asScala) + case invalidNo => Failure( ServiceException( - s"There are not enough coordinates for averaging. Found ${nearestCoords.size} within the given distance of " + + s"There are not enough coordinates for averaging. Found $invalidNo within the given distance of " + s"$maxCoordinateDistance but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." ) ) - case None => - /* Check if enough coordinates are within the coordinate distance limit */ - val nearestCoordsInMaxDistance = nearestCoords.filter(coordDistance => - coordDistance.getDistance - .isLessThan(maxCoordinateDistance) - ) - if (nearestCoordsInMaxDistance.size < amountOfInterpolationCoords) { - Failure( - ServiceException( - s"There are not enough coordinates within the max coordinate distance of $maxCoordinateDistance. Found ${nearestCoordsInMaxDistance.size} but need $amountOfInterpolationCoords. Please make sure that there are enough coordinates within the given distance." - ) - ) - } else { - /* Check, if the queried coordinate is surrounded at each quadrant */ - val (topLeft, topRight, bottomLeft, bottomRight) = nearestCoords - .map(_.getCoordinateB) - .foldLeft((false, false, false, false)) { - case ((tl, tr, bl, br), point) => - ( - tl || (point.getX < queryPoint.getX && point.getY > queryPoint.getY), - tr || (point.getX > queryPoint.getX && point.getY > queryPoint.getY), - bl || (point.getX < queryPoint.getX && point.getY < queryPoint.getY), - br || (point.getX > queryPoint.getX && point.getY < queryPoint.getY), - ) - } - - /* There has to be a coordinate in each quadrant */ - if (topLeft && topRight && bottomLeft && bottomRight) - Success(nearestCoords) - else - Failure( - ServiceException( - s"The queried point shall be surrounded by $amountOfInterpolationCoords weather coordinates, which are in each quadrant. This is not the case." - ) - ) - } } } @@ -293,137 +247,37 @@ trait WeatherSource { object WeatherSource { def apply( - dataSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource, - simulationStart: ZonedDateTime, - ): WeatherSource = - checkConfig(dataSourceConfig)(simulationStart) - - /** Check the provided weather data source configuration to ensure its - * validity. If the configuration is valid, a function to build the - * corresponding [[WeatherSource]] instance is returned. For any invalid - * configuration parameters exceptions are thrown. - * - * @param weatherDataSourceCfg - * the config to be checked - * @return - * a function that can be used to actually build the configured weather - * data source - */ - def checkConfig( weatherDataSourceCfg: SimonaConfig.Simona.Input.Weather.Datasource - ): ZonedDateTime => WeatherSource = { + )(implicit simulationStart: ZonedDateTime): WeatherSource = { + // get coordinate source + implicit val coordinateSourceFunction: IdCoordinateSource = + buildCoordinateSource(weatherDataSourceCfg.coordinateSource) - // check and get coordinate source - val coordinateSourceFunction: () => IdCoordinateSource = - checkCoordinateSource( - weatherDataSourceCfg.coordinateSource - ) - - /* Check, if the column scheme is supported */ - if (!WeatherScheme.isEligibleInput(weatherDataSourceCfg.scheme)) - throw new InvalidConfigParameterException( - s"The weather data scheme '${weatherDataSourceCfg.scheme}' is not supported. Supported schemes:\n\t${WeatherScheme.values - .mkString("\n\t")}" - ) - - // check weather source parameters - val supportedWeatherSources = - Set("influxdb1x", "csv", "sql", "couchbase", "sample") val definedWeatherSources = Vector( weatherDataSourceCfg.sampleParams, weatherDataSourceCfg.csvParams, weatherDataSourceCfg.influxDb1xParams, weatherDataSourceCfg.couchbaseParams, weatherDataSourceCfg.sqlParams, - ).filter(_.isDefined) + ).find(_.isDefined).flatten + + if (definedWeatherSources.isEmpty) { + // should not happen, due to the config fail fast check + throw new SourceException( + s"Expected a WeatherSource, but no source where defined in $weatherDataSourceCfg." + ) + } - val timestampPattern: Option[String] = weatherDataSourceCfg.timestampPattern - val scheme: String = weatherDataSourceCfg.scheme - val resolution: Option[Long] = weatherDataSourceCfg.resolution - val distance: ComparableQuantity[Length] = + implicit val resolution: Option[Long] = weatherDataSourceCfg.resolution + implicit val distance: ComparableQuantity[Length] = Quantities.getQuantity( weatherDataSourceCfg.maxCoordinateDistance, Units.METRE, ) - // check that only one source is defined - if (definedWeatherSources.size > 1) - throw new InvalidConfigParameterException( - s"Multiple weather sources defined: '${definedWeatherSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + - s"Please define only one source!\nAvailable sources:\n\t${supportedWeatherSources.mkString("\n\t")}" - ) - definedWeatherSources.headOption match { - case Some( - Some(baseCsvParams @ BaseCsvParams(csvSep, directoryPath, _)) - ) => - checkBaseCsvParams(baseCsvParams, "WeatherSource") - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - csvSep, - Paths.get(directoryPath), - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance, - )(simulationStart) - case Some(Some(params: CouchbaseParams)) => - checkCouchbaseParams(params) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance, - )(simulationStart) - case Some(Some(params @ InfluxDb1xParams(database, _, url))) => - checkInfluxDb1xParams("WeatherSource", url, database) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance, - )(simulationStart) - case Some(Some(params: SqlParams)) => - checkSqlParams(params) - (simulationStart: ZonedDateTime) => - WeatherSourceWrapper( - params, - coordinateSourceFunction, - timestampPattern, - scheme, - resolution, - distance, - )(simulationStart) - case Some(Some(_: SampleParams)) => - // sample weather, no check required - // coordinate source must be sample coordinate source - // calling the function here is not an issue as the sample coordinate source is already - // an object (= no overhead costs) - coordinateSourceFunction() match { - case _: SampleWeatherSource.SampleIdCoordinateSource.type => - // all fine - (simulationStart: ZonedDateTime) => - new SampleWeatherSource()(simulationStart) - case coordinateSource => - // cannot use sample weather source with other combination of weather source than sample weather source - throw new InvalidConfigParameterException( - s"Invalid coordinate source " + - s"'${coordinateSource.getClass.getSimpleName}' defined for SampleWeatherSource. " + - "Please adapt the configuration to use sample coordinate source for weather data!" - ) - } - case None | Some(_) => - throw new InvalidConfigParameterException( - s"No weather source defined! This is currently not supported! Please provide the config parameters for one " + - s"of the following weather sources:\n\t${supportedWeatherSources.mkString("\n\t")}" - ) - } + buildPSDMSource(weatherDataSourceCfg, definedWeatherSources) + .map(WeatherSourceWrapper.apply) + .getOrElse(new SampleWeatherSource()) } /** Check the provided coordinate id data source configuration to ensure its @@ -437,99 +291,57 @@ object WeatherSource { * a function that can be used to actually build the configured coordinate * id data source */ - private def checkCoordinateSource( + private def buildCoordinateSource( coordinateSourceConfig: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource - ): () => IdCoordinateSource = { - val supportedCoordinateSources = Set("csv", "sql", "sample") + ): IdCoordinateSource = { val definedCoordSources = Vector( coordinateSourceConfig.sampleParams, coordinateSourceConfig.csvParams, coordinateSourceConfig.sqlParams, - ).filter(_.isDefined) + ).find(_.isDefined).flatten - // check that only one source is defined - if (definedCoordSources.size > 1) - throw new InvalidConfigParameterException( - s"Multiple coordinate sources defined: '${definedCoordSources.map(_.getClass.getSimpleName).mkString("\n\t")}'." + - s"Please define only one source!\nAvailable sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" - ) - - // check source parameters - definedCoordSources.headOption match { + definedCoordSources match { case Some( - Some(baseCsvParams @ BaseCsvParams(csvSep, directoryPath, _)) + BaseCsvParams(csvSep, directoryPath, _) ) => - checkBaseCsvParams(baseCsvParams, "CoordinateSource") - val idCoordinateFactory = checkCoordinateFactory( - coordinateSourceConfig.gridModel + val idCoordinateFactory = + coordinateSourceConfig.gridModel.toLowerCase match { + case "icon" => new IconIdCoordinateFactory() + case "cosmo" => new CosmoIdCoordinateFactory() + } + + new CsvIdCoordinateSource( + idCoordinateFactory, + new CsvDataSource( + csvSep, + Paths.get(directoryPath), + new FileNamingStrategy(), + ), ) - () => - new CsvIdCoordinateSource( - idCoordinateFactory, - new CsvDataSource( - csvSep, - Paths.get(directoryPath), - new FileNamingStrategy(), - ), - ) case Some( - Some( - sqlParams @ SqlParams( - jdbcUrl, - userName, - password, - schemaName, - tableName, - ) + SqlParams( + jdbcUrl, + userName, + password, + schemaName, + tableName, ) ) => - checkSqlParams(sqlParams) - - () => - new SqlIdCoordinateSource( - new SqlConnector(jdbcUrl, userName, password), - schemaName, - tableName, - new SqlIdCoordinateFactory(), - ) + new SqlIdCoordinateSource( + new SqlConnector(jdbcUrl, userName, password), + schemaName, + tableName, + new SqlIdCoordinateFactory(), + ) case Some( - Some( - _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams - ) + _: SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams ) => // sample coordinates, no check required - () => SampleWeatherSource.SampleIdCoordinateSource - case None | Some(_) => - throw new InvalidConfigParameterException( - s"No coordinate source defined! This is currently not supported! Please provide the config parameters for one " + - s"of the following coordinate sources:\n\t${supportedCoordinateSources.mkString("\n\t")}" - ) - } - } - - /** Check the provided coordinate grid model configuration to ensure its - * validity. If the configuration is valid, the corresponding - * IdCoordinateSource is returned. For any invalid configuration parameters - * exceptions are thrown. - * - * @param gridModel - * the grid model string to be checked - * @return - * a function that can be used to actually build the id coordinate factory - * for the grid model - */ - private def checkCoordinateFactory( - gridModel: String - ): IdCoordinateFactory = { - if (gridModel.isEmpty) - throw new InvalidConfigParameterException("No grid model defined!") - gridModel.toLowerCase() match { - case "icon" => new IconIdCoordinateFactory() - case "cosmo" => new CosmoIdCoordinateFactory() - case _ => - throw new InvalidConfigParameterException( - s"Grid model '$gridModel' is not supported!" - ) + SampleWeatherSource.SampleIdCoordinateSource + case None => + throw new SourceException( + s"Expected an IdCoordinateSource, but no source where defined in $coordinateSourceConfig." + ); } } diff --git a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala index 0fe6d10480..6ccc0942bd 100644 --- a/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala +++ b/src/main/scala/edu/ie3/simona/service/weather/WeatherSourceWrapper.scala @@ -25,6 +25,8 @@ import edu.ie3.datamodel.io.source.{ IdCoordinateSource, WeatherSource => PsdmWeatherSource, } +import edu.ie3.simona.config.SimonaConfig +import edu.ie3.simona.config.SimonaConfig.BaseCsvParams import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ CouchbaseParams, InfluxDb1xParams, @@ -46,7 +48,7 @@ import edu.ie3.util.DoubleUtils.ImplicitDouble import edu.ie3.util.interval.ClosedInterval import tech.units.indriya.ComparableQuantity -import java.nio.file.Path +import java.nio.file.Paths import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import javax.measure.quantity.Length @@ -199,124 +201,102 @@ private[weather] object WeatherSourceWrapper extends LazyLogging { private val DEFAULT_RESOLUTION = 3600L def apply( - csvSep: String, - directoryPath: Path, - idCoordinateSourceFunction: () => IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, + source: PsdmWeatherSource + )(implicit + simulationStart: ZonedDateTime, + idCoordinateSource: IdCoordinateSource, resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length], - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { - val idCoordinateSource = idCoordinateSourceFunction() - val source = new CsvWeatherSource( - csvSep, - directoryPath, - new FileNamingStrategy(), - idCoordinateSource, - buildFactory(scheme, timestampPattern), - ) - logger.info( - "Successfully initiated CsvWeatherSource as source for WeatherSourceWrapper." - ) + distance: ComparableQuantity[Length], + ): WeatherSourceWrapper = { WeatherSourceWrapper( source, idCoordinateSource, resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance, + distance, ) } - def apply( - couchbaseParams: CouchbaseParams, - idCoordinateSourceFunction: () => IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length], - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { - val couchbaseConnector = new CouchbaseConnector( - couchbaseParams.url, - couchbaseParams.bucketName, - couchbaseParams.userName, - couchbaseParams.password, - ) - val idCoordinateSource = idCoordinateSourceFunction() - val source = new CouchbaseWeatherSource( - couchbaseConnector, - idCoordinateSourceFunction(), - couchbaseParams.coordinateColumnName, - couchbaseParams.keyPrefix, - buildFactory(scheme, timestampPattern), - "yyyy-MM-dd'T'HH:mm:ssxxx", - ) - logger.info( - "Successfully initiated CouchbaseWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance, - ) - } + private[weather] def buildPSDMSource( + cfgParams: SimonaConfig.Simona.Input.Weather.Datasource, + definedWeatherSources: Option[Serializable], + )(implicit + idCoordinateSource: IdCoordinateSource + ): Option[PsdmWeatherSource] = { + implicit val timestampPattern: Option[String] = + cfgParams.timestampPattern + implicit val scheme: String = cfgParams.scheme - def apply( - influxDbParams: InfluxDb1xParams, - idCoordinateSourceFunction: () => IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length], - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { - val influxDb1xConnector = - new InfluxDbConnector(influxDbParams.url, influxDbParams.database) - val idCoordinateSource = idCoordinateSourceFunction() - val source = new InfluxDbWeatherSource( - influxDb1xConnector, - idCoordinateSource, - buildFactory(scheme, timestampPattern), - ) - logger.info( - "Successfully initiated InfluxDbWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance, - ) - } + val factory = buildFactory(scheme, timestampPattern) - def apply( - sqlParams: SqlParams, - idCoordinateSourceFunction: () => IdCoordinateSource, - timestampPattern: Option[String], - scheme: String, - resolution: Option[Long], - maxCoordinateDistance: ComparableQuantity[Length], - )(implicit simulationStart: ZonedDateTime): WeatherSourceWrapper = { - val sqlConnector = new SqlConnector( - sqlParams.jdbcUrl, - sqlParams.userName, - sqlParams.password, - ) - val idCoordinateSource = idCoordinateSourceFunction() - val source = new SqlWeatherSource( - sqlConnector, - idCoordinateSource, - sqlParams.schemaName, - sqlParams.tableName, - buildFactory(scheme, timestampPattern), - ) - logger.info( - "Successfully initiated SqlWeatherSource as source for WeatherSourceWrapper." - ) - WeatherSourceWrapper( - source, - idCoordinateSource, - resolution.getOrElse(DEFAULT_RESOLUTION), - maxCoordinateDistance, - ) + val source = definedWeatherSources.flatMap { + case BaseCsvParams(csvSep, directoryPath, _) => + // initializing a csv weather source + Some( + new CsvWeatherSource( + csvSep, + Paths.get(directoryPath), + new FileNamingStrategy(), + idCoordinateSource, + factory, + ) + ) + case couchbaseParams: CouchbaseParams => + // initializing a couchbase weather source + val couchbaseConnector = new CouchbaseConnector( + couchbaseParams.url, + couchbaseParams.bucketName, + couchbaseParams.userName, + couchbaseParams.password, + ) + Some( + new CouchbaseWeatherSource( + couchbaseConnector, + idCoordinateSource, + couchbaseParams.coordinateColumnName, + couchbaseParams.keyPrefix, + factory, + "yyyy-MM-dd'T'HH:mm:ssxxx", + ) + ) + case InfluxDb1xParams(database, _, url) => + // initializing an influxDb weather source + val influxDb1xConnector = + new InfluxDbConnector(url, database) + Some( + new InfluxDbWeatherSource( + influxDb1xConnector, + idCoordinateSource, + factory, + ) + ) + case sqlParams: SqlParams => + // initializing a sql weather source + val sqlConnector = new SqlConnector( + sqlParams.jdbcUrl, + sqlParams.userName, + sqlParams.password, + ) + Some( + new SqlWeatherSource( + sqlConnector, + idCoordinateSource, + sqlParams.schemaName, + sqlParams.tableName, + factory, + ) + ) + case _ => + // no weather source is initialized + None + } + + source.foreach { source => + logger.info( + s"Successfully initialized ${source.getClass.getSimpleName} as source for WeatherSourceWrapper." + ) + } + + source } private def buildFactory(scheme: String, timestampPattern: Option[String]) = diff --git a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala index 4468746834..c38ef668f1 100644 --- a/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala +++ b/src/test/scala/edu/ie3/simona/config/ConfigFailFastSpec.scala @@ -7,7 +7,11 @@ package edu.ie3.simona.config import com.typesafe.config.ConfigFactory -import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource +import edu.ie3.simona.config.SimonaConfig.Simona.Input.Weather.Datasource.{ + CoordinateSource, + SampleParams, +} import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink import edu.ie3.simona.config.SimonaConfig.Simona.Output.Sink.{Csv, InfluxDb1x} import edu.ie3.simona.config.SimonaConfig.Simona.Powerflow.Newtonraphson @@ -1012,39 +1016,81 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { /* Checking of primary source configuration is delegated to the specific actor. Tests are placed there */ "Checking weather data sources" should { - val checkWeatherDataSource = PrivateMethod[Unit](Symbol("checkWeatherDataSource")) + val csv: BaseCsvParams = + BaseCsvParams(",", "input", isHierarchic = false) + val sample = new SampleParams(true) + + val weatherDataSource = Datasource( + CoordinateSource( + None, + "icon", + Some( + SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource + .SampleParams(true) + ), + None, + ), + None, + None, + None, + 50000d, + Some(360L), + None, + "icon", + None, + Some("yyyy-MM-dd HH:mm"), + ) + "detects invalid weather data scheme" in { - val weatherDataSource = - new SimonaConfig.Simona.Input.Weather.Datasource( - CoordinateSource( - None, - "icon", - Some( - SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource - .SampleParams(true) - ), - None, - ), - None, - None, - None, - 50000d, - Some(360L), - Some( - SimonaConfig.Simona.Input.Weather.Datasource.SampleParams(true) - ), - "this won't work", - None, - Some("yyyy-MM-dd HH:mm"), + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource( + weatherDataSource.copy(scheme = "this won't work") ) + }.getMessage shouldBe "The weather data scheme 'this won't work' is not supported. " + + "Supported schemes:\n\ticon\n\tcosmo" + } + + "detect missing source" in { intercept[InvalidConfigParameterException] { ConfigFailFast invokePrivate checkWeatherDataSource( weatherDataSource ) - }.getMessage shouldBe "The weather data scheme 'this won't work' is not supported. Supported schemes:\n\ticon\n\tcosmo" + }.getMessage should startWith( + "No weather source defined! This is currently not supported! Please provide the config parameters for " + + "one of the following weather sources:" + ) + } + + "detect too many sources" in { + val tooManySources = weatherDataSource.copy( + csvParams = Some(csv), + sampleParams = Some(sample), + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource(tooManySources) + }.getMessage should startWith("Multiple weather sources defined:") + } + + "detects sample source mismatch" in { + val csvCoordinateSource = new CoordinateSource( + csvParams = Some(csv), + gridModel = "icon", + sampleParams = None, + sqlParams = None, + ) + + val sampleMismatch = weatherDataSource.copy( + coordinateSource = csvCoordinateSource, + sampleParams = Some(sample), + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkWeatherDataSource(sampleMismatch) + }.getMessage shouldBe "Invalid coordinate source 'csv' defined for SampleWeatherSource. Please adapt the configuration to use sample coordinate source for weather data!" } } @@ -1241,6 +1287,58 @@ class ConfigFailFastSpec extends UnitSpec with ConfigTestData { } } + "Checking coordinate sources" should { + val checkCoordinateSource = + PrivateMethod[Unit](Symbol("checkCoordinateSource")) + val csvParams: BaseCsvParams = BaseCsvParams( + ",", + "input", + isHierarchic = false, + ) + val sampleParams = + new SimonaConfig.Simona.Input.Weather.Datasource.CoordinateSource.SampleParams( + true + ) + + val coordinateSource = new CoordinateSource( + csvParams = None, + gridModel = "icon", + sampleParams = None, + sqlParams = None, + ) + + "detect missing source" in { + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(coordinateSource) + }.getMessage should startWith( + "No coordinate source defined! This is currently not supported! Please provide the config parameters for one of the following coordinate sources" + ) + } + + "detect too many sources" in { + val tooManySources = coordinateSource.copy( + csvParams = Some(csvParams), + sampleParams = Some(sampleParams), + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(tooManySources) + }.getMessage should startWith("Multiple coordinate sources defined:") + } + + "detect invalid grid model" in { + val invalidGridModel = coordinateSource.copy( + csvParams = Some(csvParams), + gridModel = "invalid", + ) + + intercept[InvalidConfigParameterException] { + ConfigFailFast invokePrivate checkCoordinateSource(invalidGridModel) + }.getMessage should startWith("Grid model 'invalid' is not supported!") + } + + } + "validating the typesafe config" when { "checking the availability of pekko logger parameterization" should { val checkPekkoLoggers = PrivateMethod[Unit](Symbol("checkPekkoLoggers")) diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala index aa6fd74bc6..3aacd91ff0 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceSpec.scala @@ -6,16 +6,8 @@ package edu.ie3.simona.service.weather -import edu.ie3.datamodel.io.factory.timeseries.{ - CosmoIdCoordinateFactory, - IconIdCoordinateFactory, - IdCoordinateFactory, -} import edu.ie3.datamodel.io.source.IdCoordinateSource -import edu.ie3.simona.exceptions.{ - InvalidConfigParameterException, - ServiceException, -} +import edu.ie3.simona.exceptions.ServiceException import edu.ie3.simona.ontology.messages.services.WeatherMessage import edu.ie3.simona.service.weather.WeatherSource.{ AgentCoordinates, @@ -35,7 +27,7 @@ import java.util.Optional import javax.measure.quantity.Length import scala.jdk.CollectionConverters._ import scala.jdk.OptionConverters._ -import scala.util.{Failure, Success, Try} +import scala.util.{Failure, Success} class WeatherSourceSpec extends UnitSpec { private val coordinate0 = GeoUtils.buildPoint(51.47, 7.41) @@ -47,29 +39,17 @@ class WeatherSourceSpec extends UnitSpec { 9, ) match { case Failure(exception: ServiceException) => - exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." + exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." case _ => fail("You shall not pass!") } } "issue a ServiceException, if there are not enough coordinates in max distance available" in { DummyWeatherSource.getNearestCoordinatesWithDistances( AgentCoordinates(coordinate0.getY, coordinate0.getX), - 9, - ) match { - case Failure(exception: ServiceException) => - exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." - case _ => fail("You shall not pass!") - } - } - - "issue a ServiceException, if the queried coordinate is not surrounded by the found weather coordinates" in { - val agentCoordinates = AgentCoordinates(51.3, 7.3) - DummyWeatherSource.getNearestCoordinatesWithDistances( - agentCoordinates, - 4, + 5, ) match { case Failure(exception: ServiceException) => - exception.getMessage shouldBe "The queried point shall be surrounded by 4 weather coordinates, which are in each quadrant. This is not the case." + exception.getMessage shouldBe "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 5. Please make sure that there are enough coordinates within the given distance." case _ => fail("You shall not pass!") } } @@ -244,7 +224,7 @@ class WeatherSourceSpec extends UnitSpec { case Failure(exception: ServiceException) => exception.getMessage shouldBe "Determination of coordinate weights failed." exception.getCause shouldBe ServiceException( - "There are not enough coordinates for averaging. Found 8 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." + "There are not enough coordinates for averaging. Found 4 within the given distance of 400000 m but need 9. Please make sure that there are enough coordinates within the given distance." ) case _ => fail("You shall not pass!") } @@ -299,41 +279,6 @@ class WeatherSourceSpec extends UnitSpec { ) } } - - "return correct coordinate factory" in { - val checkCoordinateFactory = - PrivateMethod[IdCoordinateFactory](Symbol("checkCoordinateFactory")) - - val cases = Table( - ("gridModel", "expectedClass", "failureMessage"), - ( - "", - classOf[InvalidConfigParameterException], - "No grid model defined!", - ), - ("icon", classOf[IconIdCoordinateFactory], ""), - ("cosmo", classOf[CosmoIdCoordinateFactory], ""), - ( - "else", - classOf[InvalidConfigParameterException], - "Grid model 'else' is not supported!", - ), - ) - - forAll(cases) { (gridModel, expectedClass, failureMessage) => - val actual = - Try(WeatherSource invokePrivate checkCoordinateFactory(gridModel)) - - actual match { - case Success(factory) => - factory.getClass shouldBe expectedClass - - case Failure(exception) => - exception.getClass shouldBe expectedClass - exception.getMessage shouldBe failureMessage - } - } - } } } @@ -452,12 +397,13 @@ case object WeatherSourceSpec { } override def findCornerPoints( - point: Point, + coordinate: Point, distance: ComparableQuantity[Length], - ): util.List[CoordinateDistance] = { - // just a dummy implementation - getClosestCoordinates(point, 4, distance) - } + ): util.List[CoordinateDistance] = + findCornerPoints( + coordinate, + getClosestCoordinates(coordinate, 9, distance), + ) override def validate(): Unit = { /* nothing to do here */ diff --git a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala index fcad677f91..3c937e0976 100644 --- a/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala +++ b/src/test/scala/edu/ie3/simona/service/weather/WeatherSourceWrapperSpec.scala @@ -320,7 +320,7 @@ object WeatherSourceWrapperSpec { override def getSourceFields: Optional[util.Set[String]] = // only required for validation - Optional.empty + Optional.empty() override def getWeather( timeInterval: ClosedInterval[ZonedDateTime] diff --git a/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala b/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala index 50fbdd6095..28cc5fcf15 100644 --- a/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala +++ b/src/test/scala/edu/ie3/simona/test/common/input/EmInputTestData.scala @@ -32,7 +32,6 @@ import edu.ie3.simona.util.ConfigUtil import edu.ie3.util.quantities.PowerSystemUnits._ import squants.energy.Kilowatts import tech.units.indriya.quantity.Quantities -import tech.units.indriya.unit.Units._ import java.util.UUID import scala.jdk.CollectionConverters.SeqHasAsJava