Skip to content

Commit

Permalink
Merge branch 'dev' into ck/#267-adaptColumnSchemeOfCOSMOWeatherData
Browse files Browse the repository at this point in the history
# Conflicts:
#	CHANGELOG.md
  • Loading branch information
sebastian-peter committed Mar 2, 2022
2 parents 46019f2 + 605898f commit 6c97ff1
Show file tree
Hide file tree
Showing 79 changed files with 832 additions and 231 deletions.
18 changes: 13 additions & 5 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,19 @@ updates:
- johanneshiry
- t-ober
- sensarmad
- sebastian-peter
- danielfeismann
ignore:
- dependency-name: org.spockframework:spock-core
versions:
- "2.1-groovy-3.0-SNAPSHOT"
- "2.1-groovy-2.5-SNAPSHOT"
- dependency-name: org.jgrapht:jgrapht-core
versions:
- ">= 1.5.a, < 1.6"
- 2.1-groovy-2.5-SNAPSHOT
- 2.1-groovy-3.0-SNAPSHOT
- 2.2-groovy-2.5-SNAPSHOT
- 2.2-groovy-3.0-SNAPSHOT
- 2.2-groovy-4.0-SNAPSHOT
- 2.2-M1-groovy-2.5
- 2.2-M1-groovy-3.0
- 2.2-M1-groovy-4.0
- 2.2-M2-groovy-2.5
- 2.2-M2-groovy-3.0
- 2.2-M2-groovy-4.0
10 changes: 9 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- SQL time series sources (`SqlTimeSeriesSource` and `SqlTimeSeriesMappingSource`) [#467](https://github.com/ie3-institute/PowerSystemDataModel/issues/467)
- Graph with impedance weighted edges including facilities to create it [#440](https://github.com/ie3-institute/PowerSystemDataModel/issues/440)
- Introducing `SqlIndividualTimeSeriesMetaInformation` which provides sql table names [#513](https://github.com/ie3-institute/PowerSystemDataModel/issues/513)

### Fixed
- Reduced code smells [#492](https://github.com/ie3-institute/PowerSystemDataModel/issues/492)
Expand All @@ -26,6 +27,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed
- BREAKING: Transformer's no load susceptance needs to be zero or negative to pass model validation [#378](https://github.com/ie3-institute/PowerSystemDataModel/issues/378)
- All input data sets for version < 3.0.0 need to be altered!
- Deprecating (as part of [#513](https://github.com/ie3-institute/PowerSystemDataModel/issues/513)):
- `edu.ie3.datamodel.io.csv.timeseries.ColumnScheme`
- `edu.ie3.datamodel.io.csv.FileNameMetaInformation`
- `edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation`
- `edu.ie3.datamodel.io.csv.timeseries.LoadProfileTimeSeriesMetaInformation`
- `edu.ie3.datamodel.io.connectors.CsvFileConnector.CsvIndividualTimeSeriesMetaInformation`
- and related methods
- BREAKING: Comprehensive harmonization around weather sources [#267](https://github.com/ie3-institute/PowerSystemDataModel/issues/267)
- Adapted the expected column scheme
- DWD COSMO model
Expand Down Expand Up @@ -61,7 +69,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- added target temperature including tolerance boundaries to `ThermalHouseInput`

### Changed
- separated entity and file naming and introduced a new FileNamingStrategy taking an EntityNamingStrategy and a FileHierarchy as arguments
- separated entity and file naming and introduced a new FileNamingStrategy taking an EntityNamingStrategy and a FileHierarchy as arguments

### Fixed
- `CsvSystemParticipantSource#getSystemParticipants()` now correctly returns electric vehicle charging station input models [PR#370](https://github.com/ie3-institute/PowerSystemDataModel/pull/370)
Expand Down
10 changes: 5 additions & 5 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ plugins {
id 'maven-publish'
id 'signing'
id 'pmd' // code check, working on source code
id 'com.diffplug.spotless' version '6.2.2'//code format
id 'com.github.spotbugs' version '5.0.5' // code check, working on byte code
id 'com.diffplug.spotless' version '6.3.0'//code format
id 'com.github.spotbugs' version '5.0.6' // code check, working on byte code
id 'de.undercouch.download' version '5.0.1'
id 'kr.motd.sphinx' version '2.10.1' // documentation generation
id 'jacoco' // java code coverage plugin
Expand Down Expand Up @@ -58,7 +58,7 @@ dependencies {
// ie³ power system utils
implementation 'com.github.ie3-institute:PowerSystemUtils:1.6'

implementation 'tech.units:indriya:2.1.2'
implementation 'tech.units:indriya:2.1.3'

implementation 'com.github.johanneshiry:OSMonaut:v1.1.1' // tmp pbf parse

Expand All @@ -74,7 +74,7 @@ dependencies {

// testing
testImplementation 'org.junit.jupiter:junit-jupiter:5.8.2'
testImplementation 'org.spockframework:spock-core:2.1-M2-groovy-3.0'
testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0'
testImplementation 'org.objenesis:objenesis:3.2' // Mock creation with constructor parameters

// testcontainers (docker framework for testing)
Expand All @@ -92,7 +92,7 @@ dependencies {
// Databases
implementation 'org.influxdb:influxdb-java:2.22'
implementation 'com.couchbase.client:java-client:3.2.5'
runtimeOnly 'org.postgresql:postgresql:42.3.2' // postgresql jdbc driver required during runtime
runtimeOnly 'org.postgresql:postgresql:42.3.3' // postgresql jdbc driver required during runtime

implementation 'commons-io:commons-io:2.11.0' // I/O functionalities
implementation 'org.apache.commons:commons-compress:1.21' // I/O functionalities
Expand Down
67 changes: 48 additions & 19 deletions src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,10 @@
import edu.ie3.datamodel.exceptions.ConnectorException;
import edu.ie3.datamodel.io.IoUtil;
import edu.ie3.datamodel.io.csv.*;
import edu.ie3.datamodel.io.csv.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.FileNamingStrategy;
import edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation;
import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import edu.ie3.datamodel.models.UniqueEntity;
import edu.ie3.datamodel.models.timeseries.TimeSeries;
import edu.ie3.datamodel.models.timeseries.TimeSeriesEntry;
Expand Down Expand Up @@ -41,7 +42,8 @@ public class CsvFileConnector implements DataConnector {
new HashMap<>();
private final Map<UUID, BufferedCsvWriter> timeSeriesWriters = new HashMap<>();
// ATTENTION: Do not finalize. It's meant for lazy evaluation.
private Map<UUID, CsvIndividualTimeSeriesMetaInformation> individualTimeSeriesMetaInformation;
private Map<UUID, edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation>
individualTimeSeriesMetaInformation;
private final FileNamingStrategy fileNamingStrategy;
private final String baseDirectoryName;

Expand Down Expand Up @@ -219,8 +221,25 @@ public BufferedReader initReader(String filePath) throws FileNotFoundException {
*
* @param timeSeriesUuid The time series in question
* @return An option on the queried information
* @deprecated since 3.0. Use {@link #individualTimeSeriesMetaInformation(UUID)} instead
*/
public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMetaInformation(
@Deprecated(since = "3.0", forRemoval = true)
public Optional<edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation>
getIndividualTimeSeriesMetaInformation(UUID timeSeriesUuid) {
return individualTimeSeriesMetaInformation(timeSeriesUuid)
.map(edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation::new);
}

/**
* Get time series meta information for a given uuid.
*
* <p>This method lazily evaluates the mapping from <i>all</i> time series files to their meta
* information.
*
* @param timeSeriesUuid The time series in question
* @return An option on the queried information
*/
public Optional<IndividualTimeSeriesMetaInformation> individualTimeSeriesMetaInformation(
UUID timeSeriesUuid) {
if (Objects.isNull(individualTimeSeriesMetaInformation))
individualTimeSeriesMetaInformation = buildIndividualTimeSeriesMetaInformation();
Expand All @@ -233,7 +252,7 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
*
* @return Mapping from time series uuid to it's meta information.
*/
private Map<UUID, CsvIndividualTimeSeriesMetaInformation>
private Map<UUID, edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation>
buildIndividualTimeSeriesMetaInformation() {
return getIndividualTimeSeriesFilePaths().parallelStream()
.map(
Expand All @@ -242,11 +261,11 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
String filePathWithoutEnding = removeFileEnding(filePath);
IndividualTimeSeriesMetaInformation metaInformation =
(IndividualTimeSeriesMetaInformation)
fileNamingStrategy.extractTimeSeriesMetaInformation(filePathWithoutEnding);
return new CsvIndividualTimeSeriesMetaInformation(
fileNamingStrategy.timeSeriesMetaInformation(filePathWithoutEnding);
return new edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation(
metaInformation, filePathWithoutEnding);
})
.collect(Collectors.toMap(FileNameMetaInformation::getUuid, v -> v));
.collect(Collectors.toMap(TimeSeriesMetaInformation::getUuid, v -> v));
}

/**
Expand All @@ -257,7 +276,7 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
* possible readers will be initialized.
* @return A mapping from column scheme to the individual time series meta information
*/
public Map<ColumnScheme, Set<CsvIndividualTimeSeriesMetaInformation>>
public Map<ColumnScheme, Set<edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation>>
getCsvIndividualTimeSeriesMetaInformation(ColumnScheme... columnSchemes) {
return getIndividualTimeSeriesFilePaths().parallelStream()
.map(
Expand All @@ -269,7 +288,8 @@ public Optional<IndividualTimeSeriesMetaInformation> getIndividualTimeSeriesMeta
.map(Optional::get)
.collect(
Collectors.groupingBy(
CsvIndividualTimeSeriesMetaInformation::getColumnScheme, Collectors.toSet()));
edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation::getColumnScheme,
Collectors.toSet()));
}

/**
Expand Down Expand Up @@ -322,11 +342,11 @@ private Set<String> getIndividualTimeSeriesFilePaths() {
* allowed.
* @return An {@link Optional} to {@link IndividualTimeSeriesMetaInformation}
*/
private Optional<CsvIndividualTimeSeriesMetaInformation> buildCsvTimeSeriesMetaInformation(
String filePathString, ColumnScheme... columnSchemes) {
private Optional<edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation>
buildCsvTimeSeriesMetaInformation(String filePathString, ColumnScheme... columnSchemes) {
try {
FileNameMetaInformation metaInformation =
fileNamingStrategy.extractTimeSeriesMetaInformation(filePathString);
TimeSeriesMetaInformation metaInformation =
fileNamingStrategy.timeSeriesMetaInformation(filePathString);
if (!IndividualTimeSeriesMetaInformation.class.isAssignableFrom(metaInformation.getClass())) {
log.error(
"The time series file '{}' does not represent an individual time series.",
Expand All @@ -350,7 +370,7 @@ private Optional<CsvIndividualTimeSeriesMetaInformation> buildCsvTimeSeriesMetaI
return Optional.empty();
}
return Optional.of(
new CsvIndividualTimeSeriesMetaInformation(
new edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation(
individualMetaInformation.getUuid(),
individualMetaInformation.getColumnScheme(),
filePathString));
Expand Down Expand Up @@ -433,19 +453,28 @@ public void shutdown() {
});
}

/** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */
/**
* Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file
*
* @deprecated since 3.0. Use {@link
* edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation} instead
*/
@Deprecated(since = "3.0", forRemoval = true)
public static class CsvIndividualTimeSeriesMetaInformation
extends IndividualTimeSeriesMetaInformation {
extends edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation {
private final String fullFilePath;

public CsvIndividualTimeSeriesMetaInformation(
UUID uuid, ColumnScheme columnScheme, String fullFilePath) {
UUID uuid,
edu.ie3.datamodel.io.csv.timeseries.ColumnScheme columnScheme,
String fullFilePath) {
super(uuid, columnScheme);
this.fullFilePath = fullFilePath;
}

public CsvIndividualTimeSeriesMetaInformation(
IndividualTimeSeriesMetaInformation metaInformation, String fullFilePath) {
edu.ie3.datamodel.io.csv.timeseries.IndividualTimeSeriesMetaInformation metaInformation,
String fullFilePath) {
this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*
* © 2022. TU Dortmund University,
* Institute of Energy Systems, Energy Efficiency and Energy Economics,
* Research group Distribution grid planning and operation
*/
package edu.ie3.datamodel.io.csv;

import edu.ie3.datamodel.io.naming.timeseries.ColumnScheme;
import edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation;
import java.util.Objects;
import java.util.UUID;

/** Enhancing the {@link IndividualTimeSeriesMetaInformation} with the full path to csv file */
public class CsvIndividualTimeSeriesMetaInformation extends IndividualTimeSeriesMetaInformation {
private final String fullFilePath;

public CsvIndividualTimeSeriesMetaInformation(
UUID uuid, ColumnScheme columnScheme, String fullFilePath) {
super(uuid, columnScheme);
this.fullFilePath = fullFilePath;
}

public CsvIndividualTimeSeriesMetaInformation(
IndividualTimeSeriesMetaInformation metaInformation, String fullFilePath) {
this(metaInformation.getUuid(), metaInformation.getColumnScheme(), fullFilePath);
}

public String getFullFilePath() {
return fullFilePath;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof CsvIndividualTimeSeriesMetaInformation that)) return false;
if (!super.equals(o)) return false;
return fullFilePath.equals(that.fullFilePath);
}

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), fullFilePath);
}

@Override
public String toString() {
return "CsvIndividualTimeSeriesMetaInformation{"
+ "uuid="
+ getUuid()
+ ", columnScheme="
+ getColumnScheme()
+ ", fullFilePath='"
+ fullFilePath
+ '\''
+ '}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@
import java.util.Objects;
import java.util.UUID;

/** Meta information, that can be derived from a certain file name */
/**
* Meta information, that can be derived from a certain file name
*
* @deprecated since 3.0. Use {@link edu.ie3.datamodel.io.naming.TimeSeriesMetaInformation} instead
*/
@Deprecated(since = "3.0", forRemoval = true)
public abstract class FileNameMetaInformation {
private final UUID uuid;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@
import java.util.Objects;
import java.util.Optional;

/** Yet supported column schemes in individual time series */
/**
* Yet supported column schemes in individual time series
*
* @deprecated since 3.0. Use {@link edu.ie3.datamodel.io.naming.timeseries.ColumnScheme} instead
*/
@Deprecated(since = "3.0", forRemoval = true)
public enum ColumnScheme {
ENERGY_PRICE("c", EnergyPriceValue.class),
ACTIVE_POWER("p", PValue.class),
Expand All @@ -24,6 +29,7 @@ public enum ColumnScheme {
private final String scheme;
private final Class<? extends Value> valueClass;

@Deprecated
ColumnScheme(String scheme, Class<? extends Value> valueClass) {
this.scheme = scheme;
this.valueClass = valueClass;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,13 @@
import java.util.Objects;
import java.util.UUID;

/** Specific meta information, that can be derived from a individual time series file */
/**
* Specific meta information, that can be derived from a individual time series file
*
* @deprecated since 3.0. Use {@link
* edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation} instead
*/
@Deprecated(since = "3.0", forRemoval = true)
public class IndividualTimeSeriesMetaInformation extends FileNameMetaInformation {
private final ColumnScheme columnScheme;

Expand All @@ -18,6 +24,20 @@ public IndividualTimeSeriesMetaInformation(UUID uuid, ColumnScheme columnScheme)
this.columnScheme = columnScheme;
}

public IndividualTimeSeriesMetaInformation(
edu.ie3.datamodel.io.naming.timeseries.IndividualTimeSeriesMetaInformation
newMetaInformation) {
super(newMetaInformation.getUuid());
this.columnScheme =
ColumnScheme.parse(newMetaInformation.getColumnScheme().toString())
.orElseThrow(
() ->
new RuntimeException(
"Cannot convert new column scheme "
+ newMetaInformation.getColumnScheme().getScheme()
+ " to deprecated column scheme!"));
}

public ColumnScheme getColumnScheme() {
return columnScheme;
}
Expand Down
Loading

0 comments on commit 6c97ff1

Please sign in to comment.