diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6f94285f8b..e585f31ce04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,8 @@ -# .github/workflows/ci.yml - name: CI Job to Generate JUnit Reports with Diff and Allure Reports on: schedule: - - cron: '17 0 * * *' # Nightly run at 00:17 UTC + - cron: 17 0 * * * # nightly run; 17th minute to decrease odds of delayed or dropped job push: branches: - main @@ -13,29 +11,19 @@ on: - main permissions: - contents: write # Allows pushing to branches - pages: write # Allows managing GitHub Pages (requires PAT) - actions: write # Allows workflow actions - checks: write # Allows updating check runs - pull-requests: write # Allows updating pull requests + contents: write # Grant write permissions for contents + checks: write # Grant write permissions for checks, only effective on push + pull-requests: write # Explicitly grant write permissions for pull requests jobs: generate-reports: runs-on: ubuntu-latest env: JOB_TYPE: ${{ github.event_name == 'schedule' && 'nightly' || 'ci' }} - REPO_URL: https://github.com/${{ github.repository }} steps: - - name: Checkout main branch + - name: Checkout repository uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all branches and tags - - - name: Setup Git User - run: | - git config --global user.email "actions@github.com" - git config --global user.name "GitHub Actions" - name: Make Install Script Executable run: chmod +x INSTALL.sh @@ -79,106 +67,91 @@ jobs: - name: Parse Test Results run: | # Extract test IDs and their statuses into a sorted file - awk -F '|' '{print $2 "|" $3}' /tmp/SHARED.UNITS | grep -E 'PASS|FAIL' | sort > current_test_results_${{ env.JOB_TYPE }}.txt + awk -F '|' '{print $2 "|" $3}' /tmp/SHARED.UNITS | grep -E 'PASS|FAIL' | sort > current_test_results.txt - # Check out 'test-results' branch into 'test-results' subdirectory - - name: Checkout test-results branch into subdirectory - uses: actions/checkout@v4 + - name: Upload Current Test Results for Comparison + uses: actions/upload-artifact@v4 with: - repository: ${{ github.repository }} - ref: test-results - path: test-results - fetch-depth: 0 - persist-credentials: true # Ensure we can push back + name: test-results-${{ env.JOB_TYPE }} + path: current_test_results.txt - # Handle case where 'test-results' branch doesn't exist - - name: Initialize test-results branch if it doesn't exist - run: | - cd test-results - if [ "$(git rev-parse --abbrev-ref HEAD)" != "test-results" ]; then - echo "test-results branch does not exist. Creating it." - git checkout -b test-results - git commit --allow-empty -m "Initialize test-results branch" - git push origin test-results - else - echo "test-results branch already exists." - fi - cd .. + - name: Download Previous Test Results for Comparison + uses: actions/download-artifact@v4 + with: + name: test-results-${{ env.JOB_TYPE }} + path: previous-results - name: Compare Test Results id: compare_tests run: | - if [ -f "test-results/previous_test_results_${{ env.JOB_TYPE }}.txt" ]; then - if diff test-results/previous_test_results_${{ env.JOB_TYPE }}.txt current_test_results_${{ env.JOB_TYPE }}.txt > /dev/null; then - echo "No changes in test results." + if [ -f "previous-results/current_test_results.txt" ]; then + if diff previous-results/current_test_results.txt current_test_results.txt > /dev/null; then + echo "No changes in test results." echo "TEST_CHANGED=false" >> $GITHUB_ENV - else - echo "Changes detected in test results." + else + echo "Changes detected in test results." echo "TEST_CHANGED=true" >> $GITHUB_ENV fi else echo "No previous test results found. Proceeding with report generation." echo "TEST_CHANGED=true" >> $GITHUB_ENV - fi + fi continue-on-error: true - - name: Update Test Results in test-results Branch - if: always() - run: | - cp current_test_results_${{ env.JOB_TYPE }}.txt test-results/previous_test_results_${{ env.JOB_TYPE }}.txt - cd test-results - git add previous_test_results_${{ env.JOB_TYPE }}.txt - git commit -m "Update test results for ${{ env.JOB_TYPE }}" || echo "No changes to commit." - git push origin test-results - cd .. - - # Run test report generation steps always + # Continue only if tests changed - name: Run JUnit Report Generation Script + if: env.TEST_CHANGED == 'true' continue-on-error: true run: | python3 scripts/into_junit.py /tmp/SHARED.UNITS ${{ env.TIMESTAMP }} 1 > junit.xml - name: Convert JUnit XML to Standard HTML Report + if: env.TEST_CHANGED == 'true' continue-on-error: true run: | cat junit.xml junit2html junit.xml ${{ env.BASELINE_COMPAT_PATH }}/junit-standard-report.html - name: Convert JUnit XML to Matrix HTML Report + if: env.TEST_CHANGED == 'true' continue-on-error: true run: | junit2html --report-matrix ${{ env.BASELINE_COMPAT_PATH }}/junit-matrix-report.html junit.xml - name: Upload JUnit XML Report + if: env.TEST_CHANGED == 'true' continue-on-error: true uses: actions/upload-artifact@v4 with: - name: junit-report + name: junit-report-${{ env.JOB_TYPE }} path: junit.xml - name: Upload Standard HTML Report + if: env.TEST_CHANGED == 'true' continue-on-error: true uses: actions/upload-artifact@v4 with: - name: junit-standard-html-report + name: junit-standard-html-report-${{ env.JOB_TYPE }} path: ${{ env.BASELINE_COMPAT_PATH }}/junit-standard-report.html - name: Upload Matrix HTML Report + if: env.TEST_CHANGED == 'true' continue-on-error: true uses: actions/upload-artifact@v4 with: - name: junit-matrix-html-report + name: junit-matrix-html-report-${{ env.JOB_TYPE }} path: ${{ env.BASELINE_COMPAT_PATH }}/junit-matrix-report.html - name: Upload Test Output Log Files + if: env.TEST_CHANGED == 'true' continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-output-logs + name: test-output-logs-${{ env.JOB_TYPE }} path: ${{ env.BASELINE_COMPAT_PATH }} - name: Display JUnit Test Results - if: github.event_name == 'push' # Only run this step on pushes to main + if: (github.event_name == 'push') && (env.TEST_CHANGED == 'true') # Only run this step on pushes to main if there are changes uses: dorny/test-reporter@v1 with: name: 'JUnit Results' @@ -187,104 +160,99 @@ jobs: fail-on-error: false - name: Provide Report Links + if: env.TEST_CHANGED == 'true' run: | echo "JUnit reports are available as artifacts." - name: Generate environment.properties + if: env.TEST_CHANGED == 'true' run: | python scripts/generate_allure_environment.py ${{ github.sha }} ${{ github.ref_name }} > environment.properties - name: Upload environment.properties + if: env.TEST_CHANGED == 'true' uses: actions/upload-artifact@v4 with: - name: environment + name: environment-${{ env.JOB_TYPE }} path: environment.properties - # Generate Allure Report - - name: Generate Allure Report + - name: Get Allure history if: env.TEST_CHANGED == 'true' - run: | - # Install Allure if not already installed - sudo apt-get update - sudo apt-get install -y default-jre - curl -sSL https://github.com/allure-framework/allure2/releases/download/2.18.1/allure-2.18.1.tgz | sudo tar -zx -C /opt/ - sudo ln -s /opt/allure-2.18.1/bin/allure /usr/bin/allure - - # Create build/allure-results directory - mkdir -p build/allure-results - - # Copy JUnit XML report to allure results - cp junit.xml build/allure-results/ - - # Copy environment.properties to allure results - cp environment.properties build/allure-results/ - - # Copy previous allure history if it exists - if [ -d "test-results/allure-history/${{ env.JOB_TYPE }}/history" ]; then - mkdir -p build/allure-results/history - cp -r test-results/allure-history/${{ env.JOB_TYPE }}/history/* build/allure-results/history/ - fi - - # Generate Allure report - allure generate build/allure-results -o allure-report --clean + uses: actions/checkout@v4 + with: + ref: test-results + path: test-results - # Prepare allure-history directory for deployment - mkdir -p test-results/allure-history/${{ env.JOB_TYPE }} - cp -r allure-report/* test-results/allure-history/${{ env.JOB_TYPE }}/ + - name: Download JUnit XML Results + if: env.TEST_CHANGED == 'true' + uses: actions/download-artifact@v4 + with: + name: junit-report-${{ env.JOB_TYPE }} + path: build/allure-results - # Copy new history back for future runs - cp -r allure-report/history test-results/allure-history/${{ env.JOB_TYPE }}/ + - name: Include environment properties + if: env.TEST_CHANGED == 'true' + uses: actions/download-artifact@v4 + with: + name: environment-${{ env.JOB_TYPE }} + path: build/allure-results - - name: Copy JUnit HTML Reports to Allure History Directory + - name: Generate Allure Report if: env.TEST_CHANGED == 'true' - run: | - # Copy JUnit HTML reports to the allure-history directory - mkdir -p test-results/allure-history/${{ env.BASELINE_COMPAT_PATH }} - cp -f ${{ env.BASELINE_COMPAT_PATH }}/junit-standard-report.html test-results/allure-history/${{ env.BASELINE_COMPAT_PATH }}/ - cp -f ${{ env.BASELINE_COMPAT_PATH }}/junit-matrix-report.html test-results/allure-history/${{ env.BASELINE_COMPAT_PATH }}/ + uses: simple-elf/allure-report-action@master + if: always() + id: allure-report + with: + allure_results: build/allure-results + gh_pages: test-results + allure_report: allure-report + allure_history: allure-history + subfolder: ${{ env.SUBFOLDER }} + keep_reports: 120 + env: + SUBFOLDER: ${{ env.JOB_TYPE }} - - name: Copy Help Docs to Allure History Directory + - name: Copy JUnit HTML Reports to GitHub Pages Directory if: env.TEST_CHANGED == 'true' run: | - mkdir -p test-results/allure-history/help-docs/ - cp -r ./docs/* test-results/allure-history/help-docs/ + sudo chmod 777 . -R + ls -lA allure-history + mkdir -p allure-history/${{ env.BASELINE_COMPAT_PATH }} + cp -f ${{ env.BASELINE_COMPAT_PATH }}/* reports/tests_output/baseline-compat/ + cp -rf reports/* allure-history/reports/ - - name: Generate Root Index for Allure History + + - name: Copy Help Docs to GitHub Pages Directory if: env.TEST_CHANGED == 'true' run: | - echo "" > test-results/allure-history/index.html - echo "Project Reports and Documentation" >> test-results/allure-history/index.html - echo "" >> test-results/allure-history/index.html - echo "

Project Reports and Documentation

" >> test-results/allure-history/index.html - echo "" >> test-results/allure-history/index.html - echo "" >> test-results/allure-history/index.html - echo "" >> test-results/allure-history/index.html - - # Commit and Push Allure History to test-results Branch - - name: Commit and Push Allure History to test-results Branch + mkdir -p allure-history/help-docs/ + cp -r ./docs/* allure-history/help-docs/ + + - name: Generate Root Index for GitHub Pages if: env.TEST_CHANGED == 'true' run: | - cd test-results - git add allure-history - git commit -m "Update Allure history for ${{ env.JOB_TYPE }}" || echo "No changes to commit." - git push origin test-results - cd .. + echo "" > allure-history/index.html + echo "Project Reports and Documentation" >> allure-history/index.html + echo "" >> allure-history/index.html + echo "

Project Reports and Documentation

" >> allure-history/index.html + echo "" >> allure-history/index.html + echo "" >> allure-history/index.html + echo "" >> allure-history/index.html - # Deploy to GitHub Pages using the gh-pages branch - name: Deploy Allure reports, JUnit HTML reports, and help docs to GitHub Pages if: env.TEST_CHANGED == 'true' uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: test-results/allure-history personal_token: ${{ secrets.GITHUB_TOKEN }} publish_branch: test-results + publish_dir: allure-history - name: Auto-Approve the Pull Request if: github.event_name == 'pull_request_target' diff --git a/scripts/run_commit_tests.sh b/scripts/run_commit_tests.sh index a5f1cc1a24d..59ca519dc32 100755 --- a/scripts/run_commit_tests.sh +++ b/scripts/run_commit_tests.sh @@ -95,9 +95,8 @@ SHOW_ALL_OUTPUT=false # Set to false normally, true for debugging run_mettalog_tests 40 tests/baseline_compat/module-system/ # 200+ tests (~4 minutes) -run_mettalog_tests 40 tests/baseline_compat/hyperon-experimental_scripts/ -run_mettalog_tests 40 tests/baseline_compat/hyperon-mettalog_sanity/ -run_mettalog_tests 40 tests/baseline_compat/metta-morph_tests/ +#run_mettalog_tests 40 tests/baseline_compat/hyperon-experimental_scripts/ +#run_mettalog_tests 40 tests/baseline_compat/hyperon-mettalog_sanity/ # 50+ tests (~2 minutes) #run_mettalog_tests 40 tests/baseline_compat/metta-morph_tests/