Skip to content

Commit

Permalink
many smoke testing improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nolij
Nolij committed Nov 7, 2024
1 parent d2177e7 commit aad6acb
Showing 1 changed file with 98 additions and 72 deletions.
170 changes: 98 additions & 72 deletions buildSrc/src/main/kotlin/dev/nolij/zumegradle/SmokeTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import java.net.URL
import java.nio.file.Files
import java.util.*
import kotlin.io.path.*
import kotlin.math.max

fun sleep(millis: Long) {
Thread.sleep(millis)
Expand All @@ -33,6 +34,8 @@ class SmokeTest(
val extraArgs: List<String>? = null,
val dependencies: List<Pair<String, String>>? = null,
) {
val name: String = hashCode().toUInt().toString(16)

val versionString: String get() =
if (loaderVersion != null)
"${modLoader}:${mcVersion}:${loaderVersion}"
Expand All @@ -42,6 +45,7 @@ class SmokeTest(
override fun toString(): String {
val result = StringBuilder()

result.appendLine("name=${name}")
result.appendLine("modLoader=${modLoader}")
result.appendLine("mcVersion=${mcVersion}")
result.appendLine("loaderVersion=${loaderVersion}")
Expand All @@ -53,72 +57,68 @@ class SmokeTest(
}
}

private enum class ThreadState {
PENDING,
RUNNING,
private enum class ThreadStage {
SETUP,
TESTING,
COMPLETED,
}

private enum class FailureReason {
SETUP_NONZERO_EXIT_CODE,
TIMED_OUT,
READY,
PASSED,
FAILED,
TESTING_NONZERO_EXIT_CODE,
SUCCESS_LOG_MISSING,
}

private inner class Thread(val config: Config) {
private val name: String = config.hashCode().toUInt().toString(16)
private val instanceDir = "${workDir}/${name}"
private val modsDir = "${instanceDir}/mods"
private val logDir = "${instanceDir}/logs/latest.log"
private val logFile = File(logDir)
private val command: Array<String>
val instancePath = "${workDir}/${config.name}"
val modsPath = "${instancePath}/mods"
val command: Array<String>
val setupLogFile = File("${instancePath}/setup.log")
val testLogFile = File("${instancePath}/test.log")
val gameLogFile = File("${instancePath}/logs/latest.log")

private var process: Process? = null

private var startTimestamp: Long? = null

val isAlive: Boolean get() = process?.isAlive == true
val alive: Boolean get() = process?.isAlive == true
private val isTimedOut: Boolean get() =
if (isAlive)
if (alive)
System.nanoTime() - startTimestamp!! > threadTimeout
else
false

var stage: ThreadStage = ThreadStage.SETUP
private set
var failureReason: FailureReason? = null
private set
val failed: Boolean get() = failureReason != null

private var finalState: ThreadState? = null
val finished: Boolean get() = finalState != null
val state: ThreadState
get() {
return finalState ?:
if (startTimestamp == null) ThreadState.PENDING
else if (isTimedOut) ThreadState.TIMED_OUT
else if (isAlive) ThreadState.RUNNING
else ThreadState.READY
}
val ready: Boolean get() = !failed && stage == ThreadStage.SETUP
val done: Boolean get() = failed || stage == ThreadStage.COMPLETED

init {
Path(instanceDir).also { path ->
Path(instancePath).also { path ->
if (!path.exists())
path.createDirectories()
}

Path(modsDir).also { modsPath ->
Path(modsPath).also { modsPath ->
if (modsPath.exists())
modsPath.deleteRecursively()
modsPath.createDirectories()
}

Path(logDir).also { logPath ->
logPath.deleteIfExists()
logPath.parent.also { logsPath ->
if (!logsPath.exists())
logsPath.createDirectories()
}
}

if (gameLogFile.exists())
gameLogFile.delete()

config.dependencies?.forEach { (name, urlString) ->
URL(urlString).openStream().use { inputStream ->
FileOutputStream("${modsDir}/${name}.jar").use(inputStream::transferTo)
FileOutputStream("${modsPath}/${name}.jar").use(inputStream::transferTo)
}
}

Files.copy(modFile.toPath(), Path("${modsDir}/${modFile.name}"))
Files.copy(modFile.toPath(), Path("${modsPath}/${modFile.name}"))

val extraArgs = arrayListOf<String>()

Expand All @@ -136,66 +136,89 @@ class SmokeTest(
command = arrayOf(
portableMCBinary,
"--main-dir", mainDir,
"--work-dir", instanceDir,
"--work-dir", instancePath,
"start", config.versionString,
*extraArgs.toTypedArray(),
"--jvm-args=-DzumeGradle.auditAndExit=true",
)

ProcessBuilder(*command, "--dry")
.inheritIO()
if (ProcessBuilder(*command, "--dry")
.redirectOutput(setupLogFile)
.start()
.waitFor()
.waitFor() != 0) {
failureReason = FailureReason.SETUP_NONZERO_EXIT_CODE
}
}

fun start() {
if (state != ThreadState.PENDING)
error("Thread already started!")
private fun start() {
if (stage != ThreadStage.SETUP)
error("Thread already started")
else if (failed)
error("Cannot start thread which failed setup")

stage = ThreadStage.TESTING

startTimestamp = System.nanoTime()
process = ProcessBuilder(*command)
.inheritIO()
.redirectOutput(testLogFile)
.start()
}

fun update() {
fun step() {
var passed = false

when (state) {
ThreadState.TIMED_OUT -> process!!.destroyForcibly()
ThreadState.READY -> {
if (logFile.exists()) {
logFile.reader().use { reader ->
reader.forEachLine { line ->
if (line.endsWith("ZumeGradle audit passed"))
passed = true
if (stage == ThreadStage.SETUP && ready && anyAvailableThreads) {
return start()
} else if (stage == ThreadStage.TESTING && isTimedOut) {
failureReason = FailureReason.TIMED_OUT
process!!.destroyForcibly()
} else if (stage == ThreadStage.TESTING && !alive) {
if (process!!.exitValue() == 0) {
(if (gameLogFile.exists()) gameLogFile else testLogFile).reader().use { reader ->
reader.forEachLine { line ->
if (line.endsWith("ZumeGradle audit passed")) {
passed = true
}
}
}

if (!passed) {
failureReason = FailureReason.SUCCESS_LOG_MISSING
}
} else {
failureReason = FailureReason.TESTING_NONZERO_EXIT_CODE
}
else -> return
} else {
return
}

stage = ThreadStage.COMPLETED

if (passed) {
println("Smoke test passed for config:\n${config}")
finalState = ThreadState.PASSED
} else {
logger.error("Smoke test failed for config:\n${config}")
finalState = ThreadState.FAILED
}

printThreads()
}
}

private val threads = ArrayList<Thread>()
private val runningThreads: List<Thread> get() = threads.filter(Thread::alive)
private val pendingThreads: List<Thread> get() = threads.filter { thread -> !thread.done }
private val finishedThreads: List<Thread> get() = threads.filter(Thread::done)
private val passedThreads: List<Thread> get() = finishedThreads.filter { thread -> !thread.failed }
private val failedThreads: List<Thread> get() = threads.filter(Thread::failed)
private val availableThreads: Int get() = max(0, maxThreads - runningThreads.size)
private val anyAvailableThreads: Boolean get() = runningThreads.size < maxThreads

private fun printThreads() {
println("""
> TOTAL: ${threads.filter { thread -> thread.finished }.size}/${configs.size}
> RUNNING: ${threads.filter { thread -> thread.state == ThreadState.RUNNING }.size}/${maxThreads}
> PASSED: ${threads.filter { thread -> thread.state == ThreadState.PASSED }.size}
> FAILED: ${threads.filter { thread -> thread.state == ThreadState.FAILED }.size}
> TOTAL: ${finishedThreads.size}/${configs.size}
> RUNNING: ${runningThreads.size}/${maxThreads}
> PASSED: ${passedThreads.size}
> FAILED: ${failedThreads.size}
""".trimIndent())
}

Expand All @@ -208,19 +231,22 @@ class SmokeTest(
printThreads()

do {
while (threads.count { thread -> thread.isAlive } < maxThreads)
threads.firstOrNull { thread -> thread.state == ThreadState.PENDING }?.start() ?: break
threads.forEach(Thread::step)
sleep(500L)
threads.forEach(Thread::update)
} while (threads.any { thread -> !thread.finished })

val failedConfigs = threads
.filter { thread -> thread.state == ThreadState.FAILED }
.map { thread -> thread.config }
} while (threads.any { thread -> !thread.done })

if (failedConfigs.isNotEmpty()) {
logger.error("[{\n${failedConfigs.joinToString("}, {\n")}}]")
error("One or more tests failed. See logs for more details.")
if (failedThreads.isNotEmpty()) {
failedThreads.forEach { thread ->
logger.error(
"Config ${thread.config.name} failed!\n" +
"> STAGE: ${thread.stage}\n" +
"> CONFIG: {\n${thread.config}}\n" +
"> COMMAND: [${thread.command.joinToString(", ")}]\n" +
"> FAILURE REASON: ${thread.failureReason}\n" +
"> INSTANCE PATH: ${thread.instancePath}\n"
)
}
error("${failedThreads.size} smoke test config(s) failed. See logs for more details.")
}

println("All tests passed.")
Expand Down

0 comments on commit aad6acb

Please sign in to comment.