From aad6acb847e81e43ce157a32d588704ffe1bfb2b Mon Sep 17 00:00:00 2001 From: Nolij Date: Wed, 6 Nov 2024 19:14:54 -0500 Subject: [PATCH] many smoke testing improvements --- .../kotlin/dev/nolij/zumegradle/SmokeTest.kt | 170 ++++++++++-------- 1 file changed, 98 insertions(+), 72 deletions(-) diff --git a/buildSrc/src/main/kotlin/dev/nolij/zumegradle/SmokeTest.kt b/buildSrc/src/main/kotlin/dev/nolij/zumegradle/SmokeTest.kt index 69a0ed9..c121dc7 100644 --- a/buildSrc/src/main/kotlin/dev/nolij/zumegradle/SmokeTest.kt +++ b/buildSrc/src/main/kotlin/dev/nolij/zumegradle/SmokeTest.kt @@ -9,6 +9,7 @@ import java.net.URL import java.nio.file.Files import java.util.* import kotlin.io.path.* +import kotlin.math.max fun sleep(millis: Long) { Thread.sleep(millis) @@ -33,6 +34,8 @@ class SmokeTest( val extraArgs: List? = null, val dependencies: List>? = null, ) { + val name: String = hashCode().toUInt().toString(16) + val versionString: String get() = if (loaderVersion != null) "${modLoader}:${mcVersion}:${loaderVersion}" @@ -42,6 +45,7 @@ class SmokeTest( override fun toString(): String { val result = StringBuilder() + result.appendLine("name=${name}") result.appendLine("modLoader=${modLoader}") result.appendLine("mcVersion=${mcVersion}") result.appendLine("loaderVersion=${loaderVersion}") @@ -53,72 +57,68 @@ class SmokeTest( } } - private enum class ThreadState { - PENDING, - RUNNING, + private enum class ThreadStage { + SETUP, + TESTING, + COMPLETED, + } + + private enum class FailureReason { + SETUP_NONZERO_EXIT_CODE, TIMED_OUT, - READY, - PASSED, - FAILED, + TESTING_NONZERO_EXIT_CODE, + SUCCESS_LOG_MISSING, } private inner class Thread(val config: Config) { - private val name: String = config.hashCode().toUInt().toString(16) - private val instanceDir = "${workDir}/${name}" - private val modsDir = "${instanceDir}/mods" - private val logDir = "${instanceDir}/logs/latest.log" - private val logFile = File(logDir) - private val command: Array + val instancePath = "${workDir}/${config.name}" + val modsPath = "${instancePath}/mods" + val command: Array + val setupLogFile = File("${instancePath}/setup.log") + val testLogFile = File("${instancePath}/test.log") + val gameLogFile = File("${instancePath}/logs/latest.log") private var process: Process? = null - private var startTimestamp: Long? = null - val isAlive: Boolean get() = process?.isAlive == true + val alive: Boolean get() = process?.isAlive == true private val isTimedOut: Boolean get() = - if (isAlive) + if (alive) System.nanoTime() - startTimestamp!! > threadTimeout else false + + var stage: ThreadStage = ThreadStage.SETUP + private set + var failureReason: FailureReason? = null + private set + val failed: Boolean get() = failureReason != null - private var finalState: ThreadState? = null - val finished: Boolean get() = finalState != null - val state: ThreadState - get() { - return finalState ?: - if (startTimestamp == null) ThreadState.PENDING - else if (isTimedOut) ThreadState.TIMED_OUT - else if (isAlive) ThreadState.RUNNING - else ThreadState.READY - } + val ready: Boolean get() = !failed && stage == ThreadStage.SETUP + val done: Boolean get() = failed || stage == ThreadStage.COMPLETED init { - Path(instanceDir).also { path -> + Path(instancePath).also { path -> if (!path.exists()) path.createDirectories() } - Path(modsDir).also { modsPath -> + Path(modsPath).also { modsPath -> if (modsPath.exists()) modsPath.deleteRecursively() modsPath.createDirectories() } - - Path(logDir).also { logPath -> - logPath.deleteIfExists() - logPath.parent.also { logsPath -> - if (!logsPath.exists()) - logsPath.createDirectories() - } - } + + if (gameLogFile.exists()) + gameLogFile.delete() config.dependencies?.forEach { (name, urlString) -> URL(urlString).openStream().use { inputStream -> - FileOutputStream("${modsDir}/${name}.jar").use(inputStream::transferTo) + FileOutputStream("${modsPath}/${name}.jar").use(inputStream::transferTo) } } - Files.copy(modFile.toPath(), Path("${modsDir}/${modFile.name}")) + Files.copy(modFile.toPath(), Path("${modsPath}/${modFile.name}")) val extraArgs = arrayListOf() @@ -136,52 +136,68 @@ class SmokeTest( command = arrayOf( portableMCBinary, "--main-dir", mainDir, - "--work-dir", instanceDir, + "--work-dir", instancePath, "start", config.versionString, *extraArgs.toTypedArray(), "--jvm-args=-DzumeGradle.auditAndExit=true", ) - ProcessBuilder(*command, "--dry") - .inheritIO() + if (ProcessBuilder(*command, "--dry") + .redirectOutput(setupLogFile) .start() - .waitFor() + .waitFor() != 0) { + failureReason = FailureReason.SETUP_NONZERO_EXIT_CODE + } } - fun start() { - if (state != ThreadState.PENDING) - error("Thread already started!") + private fun start() { + if (stage != ThreadStage.SETUP) + error("Thread already started") + else if (failed) + error("Cannot start thread which failed setup") + + stage = ThreadStage.TESTING startTimestamp = System.nanoTime() process = ProcessBuilder(*command) - .inheritIO() + .redirectOutput(testLogFile) .start() } - fun update() { + fun step() { var passed = false - when (state) { - ThreadState.TIMED_OUT -> process!!.destroyForcibly() - ThreadState.READY -> { - if (logFile.exists()) { - logFile.reader().use { reader -> - reader.forEachLine { line -> - if (line.endsWith("ZumeGradle audit passed")) - passed = true + if (stage == ThreadStage.SETUP && ready && anyAvailableThreads) { + return start() + } else if (stage == ThreadStage.TESTING && isTimedOut) { + failureReason = FailureReason.TIMED_OUT + process!!.destroyForcibly() + } else if (stage == ThreadStage.TESTING && !alive) { + if (process!!.exitValue() == 0) { + (if (gameLogFile.exists()) gameLogFile else testLogFile).reader().use { reader -> + reader.forEachLine { line -> + if (line.endsWith("ZumeGradle audit passed")) { + passed = true } } } + + if (!passed) { + failureReason = FailureReason.SUCCESS_LOG_MISSING + } + } else { + failureReason = FailureReason.TESTING_NONZERO_EXIT_CODE } - else -> return + } else { + return } + + stage = ThreadStage.COMPLETED if (passed) { println("Smoke test passed for config:\n${config}") - finalState = ThreadState.PASSED } else { logger.error("Smoke test failed for config:\n${config}") - finalState = ThreadState.FAILED } printThreads() @@ -189,13 +205,20 @@ class SmokeTest( } private val threads = ArrayList() + private val runningThreads: List get() = threads.filter(Thread::alive) + private val pendingThreads: List get() = threads.filter { thread -> !thread.done } + private val finishedThreads: List get() = threads.filter(Thread::done) + private val passedThreads: List get() = finishedThreads.filter { thread -> !thread.failed } + private val failedThreads: List get() = threads.filter(Thread::failed) + private val availableThreads: Int get() = max(0, maxThreads - runningThreads.size) + private val anyAvailableThreads: Boolean get() = runningThreads.size < maxThreads private fun printThreads() { println(""" - > TOTAL: ${threads.filter { thread -> thread.finished }.size}/${configs.size} - > RUNNING: ${threads.filter { thread -> thread.state == ThreadState.RUNNING }.size}/${maxThreads} - > PASSED: ${threads.filter { thread -> thread.state == ThreadState.PASSED }.size} - > FAILED: ${threads.filter { thread -> thread.state == ThreadState.FAILED }.size} + > TOTAL: ${finishedThreads.size}/${configs.size} + > RUNNING: ${runningThreads.size}/${maxThreads} + > PASSED: ${passedThreads.size} + > FAILED: ${failedThreads.size} """.trimIndent()) } @@ -208,19 +231,22 @@ class SmokeTest( printThreads() do { - while (threads.count { thread -> thread.isAlive } < maxThreads) - threads.firstOrNull { thread -> thread.state == ThreadState.PENDING }?.start() ?: break + threads.forEach(Thread::step) sleep(500L) - threads.forEach(Thread::update) - } while (threads.any { thread -> !thread.finished }) - - val failedConfigs = threads - .filter { thread -> thread.state == ThreadState.FAILED } - .map { thread -> thread.config } + } while (threads.any { thread -> !thread.done }) - if (failedConfigs.isNotEmpty()) { - logger.error("[{\n${failedConfigs.joinToString("}, {\n")}}]") - error("One or more tests failed. See logs for more details.") + if (failedThreads.isNotEmpty()) { + failedThreads.forEach { thread -> + logger.error( + "Config ${thread.config.name} failed!\n" + + "> STAGE: ${thread.stage}\n" + + "> CONFIG: {\n${thread.config}}\n" + + "> COMMAND: [${thread.command.joinToString(", ")}]\n" + + "> FAILURE REASON: ${thread.failureReason}\n" + + "> INSTANCE PATH: ${thread.instancePath}\n" + ) + } + error("${failedThreads.size} smoke test config(s) failed. See logs for more details.") } println("All tests passed.")