-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into ms/#67-reduce-memory-usage
- Loading branch information
Showing
11 changed files
with
271 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package edu.ie3.simbench.io | ||
|
|
||
| import java.net.URL | ||
| import edu.ie3.simbench.config.SimbenchConfig | ||
| import java.nio.file.{Files, Paths, StandardCopyOption} | ||
| import scala.io.Source | ||
| import scala.util.matching.Regex | ||
| import com.typesafe.scalalogging.LazyLogging | ||
|
|
||
| class Extractor(simbenchConfig: SimbenchConfig) extends LazyLogging { | ||
|
|
||
| def download(): Unit = { | ||
| val url = new URL( | ||
| "https://daks.uni-kassel.de/bitstreams/b1fb0ccf-94a1-4d5e-921c-5f9fa44e5371/download" | ||
| ) | ||
| val inputStream = url.openStream() | ||
|
|
||
| try { | ||
| val downloadFolder = simbenchConfig.io.input.download.directory | ||
| val outputPath = s"$downloadFolder/simbench_datalinks.csv" | ||
| val path = Paths.get(outputPath) | ||
| Files.createDirectories(path.getParent) | ||
| Files.copy(inputStream, path, StandardCopyOption.REPLACE_EXISTING) | ||
| logger.debug(s"File downloaded successfully to $outputPath") | ||
| } finally { | ||
| inputStream.close() | ||
| } | ||
| } | ||
|
|
||
| /** Extracts a map of simbench-code to UUID from the downloaded file. | ||
| */ | ||
| def extractUUIDMap(): Map[String, String] = { | ||
| val downloadDir = simbenchConfig.io.input.download.directory | ||
| val outputPath = s"$downloadDir/simbench_datalinks.csv" | ||
| val uuidPattern: Regex = | ||
| """[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}""".r | ||
|
|
||
| val source = Source.fromFile(outputPath) | ||
| val lines = source.getLines().toList | ||
|
|
||
| val header = lines.head.split(",").map(_.trim) | ||
| val data = lines.tail | ||
|
|
||
| val codeIndex = header.indexOf("code") | ||
| val csvIndex = header.indexOf("csv") | ||
|
|
||
| if (codeIndex == -1 || csvIndex == -1) { | ||
| throw new IllegalArgumentException( | ||
| "The required columns ('code', 'csv') are missing." | ||
| ) | ||
| } | ||
|
|
||
| // Extract the map | ||
| val dataMap: Map[String, String] = data.flatMap { line => | ||
| val columns = line.split(",").map(_.trim) | ||
| if (columns.length > csvIndex) { | ||
| val code = columns(codeIndex) | ||
| val csv = columns(csvIndex) | ||
| val uuidOpt = uuidPattern.findFirstIn(csv) | ||
| uuidOpt.map(uuid => code -> uuid) | ||
| } else None | ||
| }.toMap | ||
|
|
||
| source.close() | ||
| dataMap | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.