Create smoke,regress,nightly tests; Add nightly GitHub workflow

.github/workflows/nightly.yml

@@ -0,0 +1,54 @@
+name: Regression test
+
+on:
+  schedule:
+    - cron: '30 2 * * *'
+  workflow_dispatch:
+
+jobs:
+  # check_date from: https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+  check_date:
+    runs-on: ubuntu-latest
+    name: Check latest commit
+    outputs:
+      should_run: ${{ steps.should_run.outputs.should_run }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: print latest_commit
+        run: echo ${{ github.sha }}
+
+      - id: should_run
+        continue-on-error: true
+        name: check latest commit is less than a day
+        if: ${{ github.event_name == 'schedule' }}
+        run: test -z $(git rev-list  --after="24 hours"  ${{ github.sha }}) && echo "::set-output name=should_run::false"
+  nightly:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone Github Repo Action
+      uses: actions/checkout@v4
+    - name: Setup apptainer
+      uses: eWaterCycle/[email protected]
+    - name: Get container from cache
+      id: cache-sif
+      uses: actions/cache@v3
+      with:
+        path: .singularity/image.sif
+        key: ${{ hashFiles('container.def', 'bin/.container-tag') }}
+    - name: Get gems and node files from cache
+      id: cache-bundle-npm
+      uses: actions/cache@v3
+      with:
+        path: |
+          .home/.gems
+          node_modules
+        key: ${{ hashFiles('Gemfile.lock') }}-${{ hashFiles('package-lock.json') }}
+    - if: ${{ steps.cache-sif.outputs.cache-hit != 'true' }}
+      name: Build container
+      run: ./bin/build_container
+    - name: Setup project
+      run: ./bin/setup
+    - name: Run regression
+      run: ./do test:nightly

.github/workflows/regress.yml

@@ -32,4 +32,4 @@ jobs:
     - name: Setup project
       run: ./bin/setup
     - name: Run regression
-      run: ./do regress
+      run: ./do test:regress

README.adoc

@@ -105,8 +105,8 @@ Quick start:
 
 ## examples
 
-# validate against the schema
-./do validate
+# run smoke tests
+./do test:smoke
 
 # generate all versions of ISA manual, as an Antora static website
 ./do gen:html_manual MANUAL_NAME=isa VERSIONS=all

Rakefile

@@ -55,20 +55,22 @@ namespace :serve do
   end
 end
 
-desc "Run the IDL compiler test suite"
-task :idl_test do
-  t = Minitest::TestTask.new(:lib_test)
-  t.test_globs = ["#{$root}/lib/idl/tests/test_*.rb"]
-  t.process_env
-  ruby t.make_test_cmd
-end
+namespace :test do
+  # "Run the IDL compiler test suite"
+  task :idl_compiler do
+    t = Minitest::TestTask.new(:lib_test)
+    t.test_globs = ["#{$root}/lib/idl/tests/test_*.rb"]
+    t.process_env
+    ruby t.make_test_cmd
+  end
 
-desc "Run the Ruby library test suite"
-task :lib_test do
-  t = Minitest::TestTask.new(:lib_test)
-  t.test_globs = ["#{$root}/lib/test/test_*.rb"]
-  t.process_env
-  ruby t.make_test_cmd
+  # "Run the Ruby library test suite"
+  task :lib do
+    t = Minitest::TestTask.new(:lib_test)
+    t.test_globs = ["#{$root}/lib/test/test_*.rb"]
+    t.process_env
+    ruby t.make_test_cmd
+  end
 end
 
 desc "Clean up all generated files"
@@ -77,7 +79,7 @@ task :clean do
   FileUtils.rm_rf $root / ".stamps"
 end
 
-namespace :validate do
+namespace :test do
   task :insts do
     puts "Checking instruction encodings..."
     inst_paths = Dir.glob("#{$root}/arch/inst/**/*.yaml").map { |f| Pathname.new(f) }
@@ -95,9 +97,10 @@ namespace :validate do
       progressbar.increment
       validator.validate(f)
     end
+    Rake::Task["test:insts"].invoke
     puts "All files validate against their schema"  
   end
-  task idl: ["gen:arch", "#{$root}/.stamps/arch-gen-_32.stamp", "#{$root}/.stamps/arch-gen-_64.stamp"]  do
+  task idl_model: ["gen:arch", "#{$root}/.stamps/arch-gen-_32.stamp", "#{$root}/.stamps/arch-gen-_64.stamp"]  do
     print "Parsing IDL code for RV32..."
     arch_def_32 = arch_def_for("_32")
     puts "done"
@@ -114,9 +117,6 @@ namespace :validate do
   end
 end
 
-desc "Validate the arch docs"
-task validate: ["validate:schema", "validate:idl", "validate:insts"]
-
 def insert_warning(str, from)
   # insert a warning on the second line
   lines = str.lines
@@ -274,67 +274,97 @@ namespace :gen do
   end
 end
 
-desc <<~DESC
-  Run the regression tests
-
-  These tests must pass before a commit will be allowed in the main branch on GitHub
-DESC
-task :regress do
-  Rake::Task["idl_test"].invoke
-  Rake::Task["lib_test"].invoke
-  Rake::Task["validate"].invoke
-  ENV["MANUAL_NAME"] = "isa"
-  ENV["VERSIONS"] = "all"
-  Rake::Task["gen:html_manual"].invoke
-  Rake::Task["gen:html"].invoke("generic_rv64")
-  Rake::Task["#{$root}/gen/certificate_doc/pdf/MockCertificateModel.pdf"].invoke
-  Rake::Task["#{$root}/gen/certificate_doc/pdf/MC100.pdf"].invoke
-  Rake::Task["#{$root}/gen/profile_doc/pdf/MockProfileRelease.pdf"].invoke
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVA20.pdf"].invoke
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVA22.pdf"].invoke
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVI20.pdf"].invoke
-
-  puts
-  puts "Regression test PASSED"
+namespace :test do
+  desc <<~DESC
+    Run smoke tests
+
+    These are basic but fast-running tests to check the database and tools
+  DESC
+  task :smoke do
+    Rake::Task["test:idl_compiler"].invoke
+    Rake::Task["test:lib"].invoke
+    Rake::Task["test:schema"].invoke
+    Rake::Task["test:idl_model"].invoke
+  end
+
+  desc <<~DESC
+    Run the regression tests
+
+    These tests must pass before a commit will be allowed in the main branch on GitHub
+  DESC
+  task :regress do
+    Rake::Task["test:smoke"].invoke
+
+    ENV["MANUAL_NAME"] = "isa"
+    ENV["VERSIONS"] = "all"
+    Rake::Task["gen:html_manual"].invoke
+
+    Rake::Task["gen:html"].invoke("generic_rv64")
+
+    Rake::Task["#{$root}/gen/certificate_doc/pdf/MockCertificateModel.pdf"].invoke
+    Rake::Task["#{$root}/gen/profile_doc/pdf/MockProfileRelease.pdf"].invoke
+
+    puts
+    puts "Regression test PASSED"
+  end
+
+  desc <<~DESC
+    Run the nightly regression tests
+
+    Generally, this tries to build all artifacts
+  DESC
+  task :nightly do
+    Rake::Task["regress"].invoke
+
+    Rake::Task["#{$root}/gen/certificate_doc/pdf/MC100.pdf"].invoke
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVA20.pdf"].invoke
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVA22.pdf"].invoke
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVI20.pdf"].invoke
+
+    puts
+    puts "Nightly regression test PASSED"
+  end
 end
 
-desc <<~DESC
-  Generate all certificates and profile PDFs.
-DESC
-task :cert_profile_pdfs do
-  puts "==================================="
-  puts "cert_profile_pdfs: Generating MC100"
-  puts "                   1st target"
-  puts "==================================="
-  Rake::Task["#{$root}/gen/certificate_doc/pdf/MC100.pdf"].invoke
-
-  puts "=================================================="
-  puts "cert_profile_pdfs: Generating MockCertificateModel"
-  puts "                   2nd target"
-  puts "=================================================="
-  Rake::Task["#{$root}/gen/certificate_doc/pdf/MockCertificateModel.pdf"].invoke
-
-  puts "==================================="
-  puts "cert_profile_pdfs: Generating RVA20"
-  puts "                   3rd target"
-  puts "==================================="
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVA20.pdf"].invoke
-
-  puts "==================================="
-  puts "cert_profile_pdfs: Generating RVA22"
-  puts "                   4th target"
-  puts "==================================="
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVA22.pdf"].invoke
-
-  puts "==================================="
-  puts "cert_profile_pdfs: Generating RVI20"
-  puts "                   5th target"
-  puts "==================================="
-  Rake::Task["#{$root}/gen/profile_doc/pdf/RVI20.pdf"].invoke
-
-  puts "==================================="
-  puts "cert_profile_pdfs: Generating MockProfileRelease"
-  puts "                   6th target"
-  puts "==================================="
-  Rake::Task["#{$root}/gen/profile_doc/pdf/MockProfileRelease.pdf"].invoke
+namespace :gen do
+  desc <<~DESC
+    Generate all certificates and profile PDFs.
+  DESC
+  task :cert_profile_pdfs do
+    puts "==================================="
+    puts "cert_profile_pdfs: Generating MC100"
+    puts "                   1st target"
+    puts "==================================="
+    Rake::Task["#{$root}/gen/certificate_doc/pdf/MC100.pdf"].invoke
+
+    puts "=================================================="
+    puts "cert_profile_pdfs: Generating MockCertificateModel"
+    puts "                   2nd target"
+    puts "=================================================="
+    Rake::Task["#{$root}/gen/certificate_doc/pdf/MockCertificateModel.pdf"].invoke
+
+    puts "==================================="
+    puts "cert_profile_pdfs: Generating RVA20"
+    puts "                   3rd target"
+    puts "==================================="
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVA20.pdf"].invoke
+
+    puts "==================================="
+    puts "cert_profile_pdfs: Generating RVA22"
+    puts "                   4th target"
+    puts "==================================="
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVA22.pdf"].invoke
+
+    puts "==================================="
+    puts "cert_profile_pdfs: Generating RVI20"
+    puts "                   5th target"
+    puts "==================================="
+    Rake::Task["#{$root}/gen/profile_doc/pdf/RVI20.pdf"].invoke
+
+    puts "==================================="
+    puts "cert_profile_pdfs: Generating MockProfileRelease"
+    puts "                   6th target"
+    puts "==================================="
+    Rake::Task["#{$root}/gen/profile_doc/pdf/MockProfileRelease.pdf"].invoke
+  end
 end

backends/arch_gen/tasks.rake

@@ -168,13 +168,14 @@ namespace :gen do
   end
 end
 
-namespace :validate do
-  desc "Validate that a configuration folder valid for the list of extensions it claims to implement"
-  task :cfg, [:config_name] do |_t, args|
-    raise "No config '#{args[:config_name]}' found in cfgs/" unless ($root / "cfgs" / args[:config_name]).directory?
+# TODO: Add this back once we settle on the config file format
+# namespace :validate do
+#   desc "Validate that a configuration folder valid for the list of extensions it claims to implement"
+#   task :cfg, [:config_name] do |_t, args|
+#     raise "No config '#{args[:config_name]}' found in cfgs/" unless ($root / "cfgs" / args[:config_name]).directory?
 
-    ArchGen.new(args[:config_name]).validate_params
+#     ArchGen.new(args[:config_name]).validate_params
 
-    puts "Success! The '#{args[:config_name]}' configuration passes validation checks"
-  end
-end
+#     puts "Success! The '#{args[:config_name]}' configuration passes validation checks"
+#   end
+# end