diff --git a/nix/t1/default.nix b/nix/t1/default.nix index 873db7ac3..106ac1059 100644 --- a/nix/t1/default.nix +++ b/nix/t1/default.nix @@ -51,7 +51,9 @@ lib.makeScope newScope elaborateConfigJson = configPath; elaborateConfig = builtins.fromJSON (lib.readFile configPath); - cases = innerSelf.callPackage ../../tests { verilator-emu = ip.verilator-emu; verilator-emu-trace = ip.verilator-emu-trace; }; + cases = innerSelf.callPackage ../../tests { + inherit (ip) verilator-emu verilator-emu-trace vcs-emu vcs-emu-trace; + }; # for the convenience to use x86 cases on non-x86 machines, avoiding the extra build time cases-x86 = @@ -117,8 +119,8 @@ lib.makeScope newScope }; vcs-dpi-lib = innerSelf.callPackage ../../difftest/online_vcs { }; vcs-dpi-lib-trace = vcs-dpi-lib.override { enable-trace = true; }; - vcs-emu-compiled = innerSelf.callPackage ./vcs.nix { inherit vcs-dpi-lib; rtl = vcs-emu-rtl; }; - vcs-emu-compiled-trace = innerSelf.callPackage ./vcs.nix { vcs-dpi-lib = vcs-dpi-lib-trace; rtl = vcs-emu-rtl; }; + vcs-emu = innerSelf.callPackage ./vcs.nix { inherit vcs-dpi-lib; rtl = vcs-emu-rtl; }; + vcs-emu-trace = innerSelf.callPackage ./vcs.nix { vcs-dpi-lib = vcs-dpi-lib-trace; rtl = vcs-emu-rtl; }; }; subsystem = rec { diff --git a/nix/t1/vcs.nix b/nix/t1/vcs.nix index 5d1668ec3..3b862f21f 100644 --- a/nix/t1/vcs.nix +++ b/nix/t1/vcs.nix @@ -39,6 +39,7 @@ stdenv.mkDerivation { passthru = { inherit (vcs-dpi-lib) enable-trace; + inherit vcs-fhs-env; }; shellHook = '' diff --git a/script/ci/src/Main.scala b/script/ci/src/Main.scala index 756995549..23a98e200 100644 --- a/script/ci/src/Main.scala +++ b/script/ci/src/Main.scala @@ -144,7 +144,7 @@ object Main: @main def runTests( jobs: String, - dontBail: Flag = Flag(false) + testType: String = "verilator" ): Unit = if jobs == "" then Logger.info("No test found, exiting") @@ -157,32 +157,31 @@ object Main: val Array(config, caseName) = testName.split(",") println("\n") Logger.info( - s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running test case $caseName with config $config" + s"${BOLD}[${index + 1}/${allJobs.length}]${RESET} Running VCS for test case $caseName with config $config" ) + val testAttr = testType.toLowerCase() match + case "verilator" => + s".#t1.$config.cases.$caseName.emu-result.with-offline" + case "vcs" => s".#t1.$config.cases.$caseName.emu-result.with-vcs" + case _ => Logger.fatal(s"Invalid test type ${testType}") val testResultPath = try os.Path( nixResolvePath( - s".#t1.$config.cases.$caseName.emu-result.with-offline" + testAttr, + if testType == "vcs" then Seq("--impure") else Seq() ) ) catch case _ => - Logger.error(s"Emulation for config $config, case $caseName fail") + Logger.error(s"VCS emulation for config $config, case $caseName fail") println("-" * 50) - println( - os.proc( - "nix", - "log", - s".#t1.$config.cases.$caseName.emu-result" - ).call() - .out - ) + println(os.proc("nix", "log", testAttr).call().out) println("-" * 50) Logger.fatal("Got error from emulation, exiting CI") - Logger.info("Checking RTL event with offline difftest") + Logger.info("Checking RTL event from VCS") val testSuccess = os.read(testResultPath / "offline-check-status").trim() == "0" if !testSuccess then @@ -193,10 +192,6 @@ object Main: val failedTests = findFailedTests() if failedTests.isEmpty then Logger.info(s"All tests passed") - else if dontBail.value then - Logger.error( - s"${BOLD}${failedTests.length} tests failed${RESET}" - ) else Logger.fatal( s"${BOLD}${failedTests.length} tests failed${RESET}" @@ -249,13 +244,17 @@ object Main: @arg( name = "cycle-update-file-path", doc = "specify the cycle update markdown file output path" - ) cycleUpdateFilePath: String + ) cycleUpdateFilePath: Option[String], + emuType: String = "verilator" ) = val failedTestsFile = os.Path(failedTestsFilePath, os.pwd) os.write.over(failedTestsFile, "## Failed Tests\n") - val cycleUpdateRecordFile = os.Path(cycleUpdateFilePath, os.pwd) - os.write.over(cycleUpdateRecordFile, "## Cycle Update\n") + if cycleUpdateFilePath.nonEmpty then + os.write.over( + os.Path(cycleUpdateFilePath.get, os.pwd), + "## Cycle Update\n" + ) os.walk(os.pwd / ".github" / "cases") .filter(_.last == "default.json") @@ -264,6 +263,11 @@ object Main: var cycleRecord = ujson.read(os.read(file)) Logger.info("Fetching CI results") + val resultAttr = emuType.toLowerCase() match + case "verilator" => + s".#t1.$config.cases._allEmuResult" + case "vcs" => s".#t1.$config.cases._allVCSEmuResult" + case _ => Logger.fatal(s"Invalid test type ${emuType}") val emuResultPath = os.Path(nixResolvePath(s".#t1.$config.cases._allEmuResult")) @@ -278,30 +282,34 @@ object Main: os.write.append(failedTestsFile, s"```text\n${journal}\n```\n") }) - Logger.info("Collecting cycle update info") - val perfCycleRegex = raw"total_cycles:\s(\d+)".r - val allCycleUpdates = os - .walk(emuResultPath) - .filter(path => path.last == "perf.txt") - .map(path => { - val cycle = os.read.lines(path).head match - case perfCycleRegex(cycle) => cycle.toInt - case _ => - throw new Exception("perf.txt file is not format as expected") - val caseName = path.segments.toSeq.reverse.drop(1).head - (caseName, cycle, cycleRecord.obj(caseName).num.toInt) - }) - .filter((_, newCycle, oldCycle) => newCycle != oldCycle) - .map: - case (caseName, newCycle, oldCycle) => - cycleRecord(caseName) = newCycle - if oldCycle == -1 then s"* 🆕 ${caseName}($config): NaN -> ${newCycle}" - else if oldCycle > newCycle then - s"* 🚀 $caseName($config): $oldCycle -> $newCycle" - else s"* 🐢 $caseName($config): $oldCycle -> $newCycle" - - os.write.append(cycleUpdateRecordFile, allCycleUpdates.mkString("\n")) - os.write.append(cycleUpdateRecordFile, "\n") + if cycleUpdateFilePath.nonEmpty then + Logger.info("Collecting cycle update info") + val perfCycleRegex = raw"total_cycles:\s(\d+)".r + val allCycleUpdates = os + .walk(emuResultPath) + .filter(path => path.last == "perf.txt") + .map(path => { + val cycle = os.read.lines(path).head match + case perfCycleRegex(cycle) => cycle.toInt + case _ => + throw new Exception("perf.txt file is not format as expected") + val caseName = path.segments.toSeq.reverse.drop(1).head + (caseName, cycle, cycleRecord.obj(caseName).num.toInt) + }) + .filter((_, newCycle, oldCycle) => newCycle != oldCycle) + .map: + case (caseName, newCycle, oldCycle) => + cycleRecord(caseName) = newCycle + if oldCycle == -1 then + s"* 🆕 ${caseName}($config): NaN -> ${newCycle}" + else if oldCycle > newCycle then + s"* 🚀 $caseName($config): $oldCycle -> $newCycle" + else s"* 🐢 $caseName($config): $oldCycle -> $newCycle" + + os.write.append( + os.Path(cycleUpdateFilePath.get, os.pwd), + allCycleUpdates.mkString("\n") + "\n" + ) os.write.over(file, ujson.write(cycleRecord, indent = 2)) end postCI @@ -316,14 +324,16 @@ object Main: println(ujson.write(Map("config" -> testPlans))) end generateTestPlan - def nixResolvePath(attr: String): String = + def nixResolvePath(attr: String, extraArgs: Seq[String] = Seq()): String = os.proc( - "nix", - "build", - "--no-link", - "--no-warn-dirty", - "--print-out-paths", - attr + Seq( + "nix", + "build", + "--no-link", + "--no-warn-dirty", + "--print-out-paths", + attr + ) ++ extraArgs ).call() .out .trim() diff --git a/tests/default.nix b/tests/default.nix index 009a9be32..f1755c122 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -6,6 +6,8 @@ , runCommand , verilator-emu , verilator-emu-trace +, vcs-emu +, vcs-emu-trace }: let @@ -20,7 +22,7 @@ let scope = lib.recurseIntoAttrs (lib.makeScope newScope (casesSelf: { recurseForDerivations = true; - inherit verilator-emu verilator-emu-trace; + inherit xLen vLen isFp verilator-emu verilator-emu-trace vcs-emu vcs-emu-trace; makeEmuResult = casesSelf.callPackage ./make-emu-result.nix { }; @@ -48,8 +50,6 @@ let stdenv = rv32-stdenv; - inherit xLen vLen isFp; - mlir = casesSelf.callPackage ./mlir { }; intrinsic = casesSelf.callPackage ./intrinsic { }; asm = casesSelf.callPackage ./asm { }; @@ -88,6 +88,25 @@ let in runCommand "catch-${configName}-all-emu-result-for-ci" { } script; + _allVCSEmuResult = + let + testPlan = builtins.fromJSON (lib.readFile ../.github/cases/${configName}/default.json); + # flattern the attr set to a list of test case derivations + # AttrSet (AttrSet Derivation) -> List Derivation + allCases = lib.filter (val: lib.isDerivation val && lib.hasAttr val.pname testPlan) + (lib.concatLists (map lib.attrValues (lib.attrValues scopeStripped))); + script = '' + mkdir -p $out + '' + (lib.concatMapStringsSep "\n" + (caseDrv: '' + _caseOutDir=$out/${caseDrv.pname} + mkdir -p "$_caseOutDir" + cp ${caseDrv.emu-result.with-vcs}/offline-check-* "$_caseOutDir"/ + '') + allCases); + in + runCommand "catch-${configName}-all-vcs-emu-result-for-ci" { } script; + all = let allCases = lib.filter lib.isDerivation @@ -104,4 +123,4 @@ let in runCommand "build-all-testcases" { } script; in -lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult; }) +lib.recurseIntoAttrs (scopeStripped // { inherit all _allEmuResult _allVCSEmuResult; }) diff --git a/tests/make-emu-result.nix b/tests/make-emu-result.nix index 129de6c0f..514a67bbb 100644 --- a/tests/make-emu-result.nix +++ b/tests/make-emu-result.nix @@ -5,6 +5,8 @@ , zstd , verilator-emu , verilator-emu-trace +, vcs-emu +, vcs-emu-trace , elaborateConfigJson }: @@ -28,6 +30,7 @@ let "--log-level" "ERROR" ]; + rtlEventOutPath = "${placeholder "out"}/${testCase.pname}-rtl-event.jsonl"; buildPhase = '' runHook preBuild @@ -36,7 +39,7 @@ let echo "[nix] Running test case ${testCase.pname} with args $difftestArgs" - RUST_BACKTRACE=full "$difftestDriver" $difftestArgs 2> $out/rtl-event.jsonl + RUST_BACKTRACE=full "$difftestDriver" $difftestArgs 2> "$rtlEventOutPath" echo "[nix] online driver done" @@ -47,15 +50,15 @@ let checkPhase = '' runHook preCheck - if [ ! -r $out/rtl-event.jsonl ]; then - echo -e "[nix] \033[0;31mInternal Error\033[0m: no rtl-event.jsonl found in output" + if [ ! -r "$rtlEventOutPath" ]; then + echo -e "[nix] \033[0;31mInternal Error\033[0m: no $rtlEventOutPath found in output" exit 1 fi - if ! jq --stream -c -e '.[]' "$out/rtl-event.jsonl" >/dev/null 2>&1; then - echo -e "[nix] \033[0;31mInternal Error\033[0m: invalid JSON file rtl-event.jsonl, showing original file:" + if ! jq --stream -c -e '.[]' "$rtlEventOutPath" >/dev/null 2>&1; then + echo -e "[nix] \033[0;31mInternal Error\033[0m: invalid JSON file $rtlEventOutPath, showing original file:" echo "--------------------------------------------" - cat $out/rtl-event.jsonl + cat $rtlEventOutPath echo "--------------------------------------------" exit 1 fi @@ -67,17 +70,20 @@ let runHook preInstall echo "[nix] compressing event log" - zstd $out/rtl-event.jsonl -o $out/rtl-event.jsonl.zstd - rm $out/rtl-event.jsonl + zstd $rtlEventOutPath -o $rtlEventOutPath.zstd + rm $rtlEventOutPath - mv perf.txt $out/ + if [ -r perf.txt ]; then + mv perf.txt $out/ + fi runHook postInstall ''; passthru.with-trace = self.overrideAttrs (old: { - difftestDriver = "${verilator-emu-trace}/bin/online_drive"; - difftestArgs = old.difftestArgs ++ [ "--wave-path" "${placeholder "out"}/wave.fst" ]; + name = old.name + "-with-trace"; + emuDriver = "${verilator-emu-trace}/bin/online_drive"; + emuDriverArgs = old.emuDriverArgs ++ [ "--wave-path" "${placeholder "out"}/wave.fst" ]; postCheck = '' if [ ! -r "$out/wave.fst" ]; then echo -e "[nix] \033[0;31mInternal Error\033[0m: waveform not found in output" @@ -87,15 +93,91 @@ let }); passthru.with-offline = self.overrideAttrs (old: { + name = old.name + "-with-offline"; preInstall = '' set +e "${verilator-emu}/bin/offline" \ --elf-file ${testCase}/bin/${testCase.pname}.elf \ - --log-file $out/rtl-event.jsonl \ + --log-file $rtlEventOutPath \ + --log-level ERROR &> $out/offline-check-journal + printf "$?" > $out/offline-check-status + set -e + ''; + }); + + passthru.with-vcs = self.overrideAttrs (old: { + name = old.name + "-with-vcs"; + __noChroot = true; + + buildPhase = '' + runHook preBuild + + mkdir -p "$out" + echo "[nix] Running VCS for ${testCase.pname}" + + RUST_BACKTRACE=full "${vcs-emu}/bin/t1-vcs-simulator" \ + --elf-file ${testCase}/bin/${testCase.pname}.elf \ + 1> /dev/null \ + 2> $rtlEventOutPath + + echo "[nix] VCS emu done" + + runHook postBuild + ''; + + postCheck = '' + set +e + + "${verilator-emu}/bin/offline" \ + --elf-file ${testCase}/bin/${testCase.pname}.elf \ + --log-file $rtlEventOutPath \ + --log-level ERROR &> $out/offline-check-journal + printf "$?" > $out/offline-check-status + + set -e + ''; + }); + + # TODO: We should write some framework like NixOS module to overlay these attribute, instead + # of override attribute one by one. + passthru.with-vcs-trace = self.overrideAttrs (old: { + name = old.name + "-with-vcs-trace"; + __noChroot = true; + buildPhase = '' + runHook preBuild + + mkdir -p "$out" + echo "[nix] Running VCS(TRACE) for ${testCase.pname}" + + RUST_BACKTRACE=full "${vcs-emu-trace}/bin/t1-vcs-simulator" \ + --elf-file ${testCase}/bin/${testCase.pname}.elf \ + --wave-path ${testCase.pname}.fsdb \ + 1> /dev/null \ + 2> $rtlEventOutPath + + echo "[nix] VCS emu done" + + runHook postBuild + ''; + + postCheck = '' + set +e + + "${verilator-emu}/bin/offline" \ + --elf-file ${testCase}/bin/${testCase.pname}.elf \ + --log-file $rtlEventOutPath \ --log-level ERROR &> $out/offline-check-journal printf "$?" > $out/offline-check-status + set -e ''; + + postInstall = '' + # VCS have weird behavior on file creation, it will report read-only filesystem on our output, + # while other tools can mutate file system correctly. + cp ${testCase.pname}.fsdb "$out" + cp -r ${vcs-emu-trace}/lib/t1-vcs-simulator.daidir "$out" + ''; }); }; in