Add option to use local files instead of download.

.gitignore

@@ -153,4 +153,4 @@ testData/download
 !testData/download/targetFolderExistsAndIsFile.zip
 
 # Downloader productive folder
-inputData/download
+input/download

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Option to use local files instead of download [#256](https://github.com/ie3-institute/simBench2psdm/issues/256)
+
+
 ## [1.0.0] - 2021-08-03
 ### Added
 -   Basic functionality to convert SimBench data sets to [PowerSystemDataModel](https://github.com/ie3-institute/powersystemdatamodel)

gradle/scripts/tscfg.gradle

@@ -5,7 +5,7 @@ task genConfigClass {
 	doLast {
 		def tscfgJarFile = project.file('build/tscfg-' + tscfgVersion + '.jar')
 		if (!tscfgJarFile.exists() || !tscfgJarFile.isFile()) {
-			download {
+			download.run {
 				src 'https://github.com/carueda/tscfg/releases/download/v' + tscfgVersion + '/tscfg-' + tscfgVersion + '.jar'
 				dest buildDir
 			}

input/config/localFileSourceConfig.conf

@@ -0,0 +1,34 @@
+# Simple config to convert only one simple lv grid
+io {
+  input {
+    folder = "input/local/"
+
+    localFile = {
+        isZipped = true
+    }
+
+    csv = {
+      fileEncoding = "UTF-8"
+      fileEnding = ".csv"
+      separator = ";"
+      directoryHierarchy = false
+    }
+  }
+
+  output {
+    csv = {
+      fileEncoding = "UTF-8"
+      fileEnding = ".csv"
+      separator = ";"
+      directoryHierarchy = false
+    }
+
+    targetFolder = "convertedData/local/"
+    compress = false
+  }
+
+  # code for local files
+  simbenchCodes = [
+    "1-LV-rural1--0-no_sw"
+  ]
+}

src/main/resources/config-template.conf

@@ -9,14 +9,23 @@ CsvConfig {
 io {
   simbenchCodes = ["String"]
   input {
-    download.baseUrl = "String" | "https://daks.uni-kassel.de/bitstreams"
-    download.folder = "String" | "inputData/download/"
-    download.failOnExistingFiles = "Boolean" | false
+    directory = "String" | "input/download/"
+
+    #@optional
+    localFile {
+        isZipped = "Boolean" | true
+        failOnExistingFiles = "Boolean" | false
+    }
+    #@optional
+    download {
+        baseUrl = "String" | "https://daks.uni-kassel.de/bitstreams"
+        failOnExistingFiles = "Boolean" | false
+    }
     csv = CsvConfig
   }
   output {
     csv = CsvConfig
-    targetFolder = "String" | "convertedData"
+    targetDir = "String" | "convertedData"
     compress = "Boolean" | true
   }
 }

src/main/scala/edu/ie3/simbench/config/ConfigValidator.scala

@@ -36,6 +36,7 @@ case object ConfigValidator {
   @throws[SimbenchException]
   private def checkValidity(io: SimbenchConfig.Io): Unit = {
     checkSimbenchCodes(io.simbenchCodes)
+    checkFileSource(io.input)
   }
 
   /** Checks the validity of the provided codes with the help of the permissible
@@ -47,11 +48,33 @@ case object ConfigValidator {
     */
   @throws[CodeValidationException]
   private def checkSimbenchCodes(codes: List[java.lang.String]): Unit = {
+    if (codes.isEmpty) {
+      throw new SimbenchConfigException(s"No simbench codes were provided!")
+    }
+
     for (code <- codes) {
       SimbenchCode.isValid(code) match {
         case Success(_)         =>
         case Failure(exception) => throw exception
       }
     }
   }
+
+  @throws[SimbenchException]
+  private def checkFileSource(cfg: SimbenchConfig.Io.Input): Unit = {
+    val sources = Vector(
+      cfg.localFile,
+      cfg.download
+    ).filter(_.isDefined)
+
+    sources.size match {
+      case 0 =>
+        throw new SimbenchConfigException(s"No file sources defined in: $cfg")
+      case 1 =>
+      case n =>
+        throw new SimbenchConfigException(
+          s"Too many file sources ($n) defined in: $cfg"
+        )
+    }
+  }
 }

src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala

@@ -1,4 +1,4 @@
-// generated by tscfg 0.9.986 on Mon Aug 09 20:12:24 CEST 2021
+// generated by tscfg 1.1.3 on Mon Dec 09 08:47:34 CET 2024
 // source: src/main/resources/config-template.conf
 
 package edu.ie3.simbench.config
@@ -47,9 +47,8 @@
         $tsCfgValidator: $TsCfgValidator
     ): SimbenchConfig.Conversion = {
       SimbenchConfig.Conversion(
-        removeSwitches = c.hasPathOrNull("removeSwitches") && c.getBoolean(
-          "removeSwitches"
-        )
+        removeSwitches =
+          c.hasPathOrNull("removeSwitches") && c.getBoolean("removeSwitches")
       )
     }
   }
@@ -62,13 +61,14 @@
   object Io {
     final case class Input(
         csv: SimbenchConfig.CsvConfig,
-        download: SimbenchConfig.Io.Input.Download
+        directory: java.lang.String,
+        download: scala.Option[SimbenchConfig.Io.Input.Download],
+        localFile: scala.Option[SimbenchConfig.Io.Input.LocalFile]
     )
     object Input {
       final case class Download(
           baseUrl: java.lang.String,
-          failOnExistingFiles: scala.Boolean,
-          directory: java.lang.String
+          failOnExistingFiles: scala.Boolean
       )
       object Download {
         def apply(
@@ -80,13 +80,30 @@
             baseUrl =
               if (c.hasPathOrNull("baseUrl")) c.getString("baseUrl")
               else "https://daks.uni-kassel.de/bitstreams",
+            failOnExistingFiles =
+              c.hasPathOrNull("failOnExistingFiles") && c.getBoolean(
+                "failOnExistingFiles"
+              )
+          )
+        }
+      }
+
+      final case class LocalFile(
+          failOnExistingFiles: scala.Boolean,
+          isZipped: scala.Boolean
+      )
+      object LocalFile {
+        def apply(
+            c: com.typesafe.config.Config,
+            parentPath: java.lang.String,
+            $tsCfgValidator: $TsCfgValidator
+        ): SimbenchConfig.Io.Input.LocalFile = {
+          SimbenchConfig.Io.Input.LocalFile(
             failOnExistingFiles =
               c.hasPathOrNull("failOnExistingFiles") && c.getBoolean(
                 "failOnExistingFiles"
               ),
-            directory =
-              if (c.hasPathOrNull("folder")) c.getString("folder")
-              else "inputData/download/"
+            isZipped = !c.hasPathOrNull("isZipped") || c.getBoolean("isZipped")
           )
         }
       }
@@ -103,12 +120,29 @@
             parentPath + "csv.",
             $tsCfgValidator
           ),
-          download = SimbenchConfig.Io.Input.Download(
-            if (c.hasPathOrNull("download")) c.getConfig("download")
-            else com.typesafe.config.ConfigFactory.parseString("download{}"),
-            parentPath + "download.",
-            $tsCfgValidator
-          )
+          directory =
+            if (c.hasPathOrNull("directory")) c.getString("directory")
+            else "input/download/",
+          download =
+            if (c.hasPathOrNull("download"))
+              scala.Some(
+                SimbenchConfig.Io.Input.Download(
+                  c.getConfig("download"),
+                  parentPath + "download.",
+                  $tsCfgValidator
+                )
+              )
+            else None,
+          localFile =
+            if (c.hasPathOrNull("localFile"))
+              scala.Some(
+                SimbenchConfig.Io.Input.LocalFile(
+                  c.getConfig("localFile"),
+                  parentPath + "localFile.",
+                  $tsCfgValidator
+                )
+              )
+            else None
         )
       }
     }
@@ -133,7 +167,7 @@
             $tsCfgValidator
           ),
           targetDir =
-            if (c.hasPathOrNull("targetFolder")) c.getString("targetFolder")
+            if (c.hasPathOrNull("targetDir")) c.getString("targetDir")
             else "convertedData"
         )
       }
@@ -208,7 +242,7 @@
     java.lang.String.valueOf(cv.unwrapped())
   }
 
-  private final class $TsCfgValidator {
+  final class $TsCfgValidator {
     private val badPaths =
       scala.collection.mutable.ArrayBuffer[java.lang.String]()
 

src/main/scala/edu/ie3/simbench/io/Extractor.scala

@@ -7,7 +7,7 @@ import scala.io.Source
 import scala.util.matching.Regex
 import com.typesafe.scalalogging.LazyLogging
 
-class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging {
+class Extractor(simbenchConfig: SimbenchConfig.Io.Input) extends LazyLogging {
 
   def download(): Unit = {
     val url = new URL(
@@ -16,7 +16,7 @@ class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging {
     val inputStream = url.openStream()
 
     try {
-      val downloadFolder = simbenchConfig.io.input.download.directory
+      val downloadFolder = simbenchConfig.directory
       val outputPath = s"$downloadFolder/simbench_datalinks.csv"
       val path = Paths.get(outputPath)
       Files.createDirectories(path.getParent)
@@ -30,7 +30,7 @@ class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging {
   /** Extracts a map of simbench-code to UUID from the downloaded file.
     */
   def extractUUIDMap(): Map[String, String] = {
-    val downloadDir = simbenchConfig.io.input.download.directory
+    val downloadDir = simbenchConfig.directory
     val outputPath = s"$downloadDir/simbench_datalinks.csv"
     val uuidPattern: Regex =
       """[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}""".r

src/main/scala/edu/ie3/simbench/io/SimbenchReader.scala

@@ -87,6 +87,12 @@ final case class SimbenchReader(
      * Await is okay here */
     val modelClassToRawData = getFieldToValueMaps
 
+    if (modelClassToRawData.forall(_._2.isEmpty)) {
+      throw SimbenchDataModelException(
+        s"Model is empty. Please check the provided data."
+      )
+    }
+
     /* Extracting all profiles */
     val loadProfiles = buildModels(modelClassToRawData, LoadProfile)
     val powerPlantProfiles = buildModels(modelClassToRawData, PowerPlantProfile)
@@ -330,7 +336,7 @@ final case class SimbenchReader(
               read(clazz, fields)
             }
         ),
-        Duration("10 s")
+        Duration("30 s")
       )
       .toMap
   }