Skip to content

Commit

Permalink
Added experiment run metadata.
Browse files Browse the repository at this point in the history
  • Loading branch information
@dominikmaeckel
dominikmaeckel committed Nov 4, 2024
1 parent ed7b99a commit b6da6d8
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 14 deletions.
17 changes: 17 additions & 0 deletions stars-core/src/main/kotlin/tools/aqua/stars/core/evaluation/TSCEvaluation.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package tools.aqua.stars.core.evaluation

import java.time.LocalDateTime
import java.util.logging.Logger
import kotlin.time.measureTime
import tools.aqua.stars.core.computeWhile
Expand All @@ -31,6 +32,11 @@ import tools.aqua.stars.core.metric.serialization.SerializableResultComparison.C
import tools.aqua.stars.core.metric.serialization.extensions.compareToBaselineResults
import tools.aqua.stars.core.metric.serialization.extensions.compareToPreviousResults
import tools.aqua.stars.core.metric.serialization.extensions.writeSerializedResults
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.applicationStartTimeString
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.comparedResultsFolder
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.logFolder
import tools.aqua.stars.core.metric.utils.ApplicationConstantsHolder.serializedResultsFolder
import tools.aqua.stars.core.metric.utils.saveAsJsonFiles
import tools.aqua.stars.core.tsc.TSC
import tools.aqua.stars.core.tsc.instance.TSCInstanceNode
Expand Down Expand Up @@ -240,6 +246,8 @@ class TSCEvaluation<
logInfo("The evaluation of all segments took: $segmentsEvaluationTime")
}
logInfo("The whole evaluation took: $totalEvaluationTime")
ApplicationConstantsHolder.totalEvaluationTime += totalEvaluationTime
ApplicationConstantsHolder.experimentEndTime = LocalDateTime.now()

postEvaluate()
}
Expand Down Expand Up @@ -302,6 +310,7 @@ class TSCEvaluation<
logFine("The evaluation of all TSCs for segment '$segment' took: $allTSCEvaluationTime")
}
logFine("The evaluation of segment '$segment' took: $segmentEvaluationTime")
ApplicationConstantsHolder.totalSegmentEvaluationTime += segmentEvaluationTime

return true
}
Expand Down Expand Up @@ -332,15 +341,21 @@ class TSCEvaluation<

val serializableMetrics = metricProviders.filterIsInstance<Serializable>()
if (serializableMetrics.any()) {
ApplicationConstantsHolder.writeMetaInfo("$logFolder/$applicationStartTimeString/")

// Write JSON files of all Serializable metrics
if (writeSerializedResults) {
println("Writing serialized results")
ApplicationConstantsHolder.writeMetaInfo(
"$serializedResultsFolder/$applicationStartTimeString/")
serializableMetrics.forEach { t -> t.writeSerializedResults() }
}

// Compare the results to the baseline
if (compareToBaselineResults) {
println("Comparing to baseline")
ApplicationConstantsHolder.writeMetaInfo(
"$comparedResultsFolder/$applicationStartTimeString/")
serializableMetrics.compareToBaselineResults().let {
resultsReproducedFromBaseline = it.noMismatch()

Expand All @@ -351,6 +366,8 @@ class TSCEvaluation<
// Compare the results to the latest run
if (compareToPreviousRun) {
println("Comparing to previous run")
ApplicationConstantsHolder.writeMetaInfo(
"$comparedResultsFolder/$applicationStartTimeString/")
serializableMetrics.compareToPreviousResults().let {
resultsReproducedFromPreviousRun = it.noMismatch()

Expand Down
79 changes: 65 additions & 14 deletions stars-core/src/main/kotlin/tools/aqua/stars/core/metric/utils/ApplicationConstantsHolder.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.logging.LogManager
import java.util.logging.Logger
import kotlin.time.Duration
import kotlinx.serialization.json.Json

/**
Expand All @@ -30,7 +31,16 @@ import kotlinx.serialization.json.Json
*/
object ApplicationConstantsHolder {
/** Holds the [LocalDateTime] at the start of the application. */
private val applicationStartTime: LocalDateTime = LocalDateTime.now()
val applicationStartTime: LocalDateTime = LocalDateTime.now()

/** Holds the [LocalDateTime] at the end of the experiment. */
var experimentEndTime: LocalDateTime? = null

/** Holds the total time spent on evaluation. */
var totalEvaluationTime: Duration = Duration.ZERO

/** Holds the total time spent on segment evaluation. */
var totalSegmentEvaluationTime: Duration = Duration.ZERO

/** Holds the [LocalDateTime] at the start of the application in the yyyy-MM-dd-HH-mm format. */
val applicationStartTimeString: String =
Expand All @@ -54,26 +64,41 @@ object ApplicationConstantsHolder {
/** Folder directory for serialized compared results produced in evaluation. */
private const val COMPARED_RESULTS_FOLDER = "compared-results"

/** File name for the metadata file. */
private const val METADATA_FILE_NAME = "experiment_run_metadata"

/** Folder directory for serialized baseline result data set. */
var baselineDirectory = "baseline"

/** May hold the command that was used to execute the application. */
var executionCommand: String = ""

/** Folder directory for serialized previous evaluation result. */
const val PREVIOUS_EVALUATION_SERIALIZED_RESULT_IDENTIFIER = "previous-evaluation"

/** Indicates whether the application is running in test mode. */
val isTestRun: Boolean
get() =
try {
Class.forName("org.junit.jupiter.api.Test")
true
} catch (_: ClassNotFoundException) {
false
}

/** Holds the folder name for the logs. */
val logFolder: String
get() = if (isTestRun()) TEST_LOG_FOLDER else ANALYSIS_LOG_FOLDER
val logFolder: String = if (isTestRun) TEST_LOG_FOLDER else ANALYSIS_LOG_FOLDER

/** Holds the [MutableList] of all currently registered [Logger]s. */
val activeLoggers: MutableList<Logger> = mutableListOf()

/** Holds the folder name for the logs. */
val serializedResultsFolder: String
get() = if (isTestRun()) "test-$SERIALIZED_RESULTS_FOLDER" else SERIALIZED_RESULTS_FOLDER
val serializedResultsFolder: String =
if (isTestRun) "test-$SERIALIZED_RESULTS_FOLDER" else SERIALIZED_RESULTS_FOLDER

/** Holds the folder name for the logs. */
val comparedResultsFolder: String
get() = if (isTestRun()) "test-$COMPARED_RESULTS_FOLDER" else COMPARED_RESULTS_FOLDER
val comparedResultsFolder: String =
if (isTestRun) "test-$COMPARED_RESULTS_FOLDER" else COMPARED_RESULTS_FOLDER

/** Holds the [Json] configuration that is used throughout the project. */
val jsonConfiguration = Json {
Expand All @@ -97,12 +122,38 @@ object ApplicationConstantsHolder {
})
}

/** Indicates whether the application is running in test mode. */
private fun isTestRun(): Boolean =
try {
Class.forName("org.junit.jupiter.api.Test")
true
} catch (_: ClassNotFoundException) {
false
/**
* Writes the experiment run metadata into the file [METADATA_FILE_NAME] in the serialization
* folders
*/
fun writeMetaInfo(directory: String): File =
File("$directory$METADATA_FILE_NAME.txt").apply {
parentFile.mkdirs()
createNewFile()
writeText(
"""
====================================================================================================
Experiment run metadata
====================================================================================================
Experiment start : ${applicationStartTime.format(DateTimeFormatter.ofPattern("dd.MM.yyyy HH:mm:ss"))}
Experiment end : ${experimentEndTime?.format(DateTimeFormatter.ofPattern("dd.MM.yyyy HH:mm:ss")) ?: "Not finished"}
Total evaluation time : $totalEvaluationTime
Total segment evaluation time : $totalSegmentEvaluationTime
Execution command : ${executionCommand
.prependIndent(" ").trimStart()}
====================================================================================================
System information
====================================================================================================
User : ${System.getProperty("user.name")}
Hostname : ${java.net.InetAddress.getLocalHost().hostName}
OS : ${System.getProperty("os.name")} ${System.getProperty("os.arch")} ${System.getProperty("os.version")}
CPU : ${Runtime.getRuntime().availableProcessors()} cores
Memory : ${Runtime.getRuntime().maxMemory() / 1024 / 1024} MB
Java version : ${System.getProperty("java.version")}
Java vendor : ${System.getProperty("java.vendor")}
"""
.trimIndent())
}
}

0 comments on commit b6da6d8

Please sign in to comment.