Merge release v2.0 into main (#876)

.github/workflows/package.yml

@@ -17,7 +17,7 @@ jobs:
     permissions: {}
     name: Package
     runs-on: ubuntu-22.04
-    timeout-minutes: 5
+    timeout-minutes: 10
     steps:
       - name: Checkout source
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/testing.yml

@@ -10,7 +10,7 @@ jobs:
   testrun_baseline:
     permissions: {}
     name:  Baseline
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 20
     steps:
       - name: Checkout source
@@ -29,7 +29,7 @@ jobs:
   testrun_api:
     permissions: {}
     name:  API
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
       - name: Checkout source
@@ -58,7 +58,7 @@ jobs:
   testrun_unit:
     permissions: {}
     name:  Unit
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 15
     steps:
       - name: Checkout source
@@ -73,9 +73,40 @@ jobs:
         shell: bash {0}
         run: cmd/build
         timeout-minutes: 10
-      - name: Run tests
+      - name: Run tests for conn module
+        shell: bash {0}
+        run: bash testing/unit/run_test_module.sh conn captures ethtool output
+      - name: Run tests for dns module
+        shell: bash {0}
+        run: bash testing/unit/run_test_module.sh dns captures reports output
+      - name: Run tests for ntp module
+        shell: bash {0}
+        run: bash testing/unit/run_test_module.sh ntp captures reports output
+      - name: Run tests for protocol module
+        shell: bash {0}
+        run: bash testing/unit/run_test_module.sh protocol captures output        
+      - name: Run tests for services module
         shell: bash {0}
-        run: bash testing/unit/run.sh
+        run: bash testing/unit/run_test_module.sh services reports results output  
+      - name: Run tests for tls module
+        shell: bash {0}
+        run: bash testing/unit/run_test_module.sh tls captures certAuth certs reports root_certs output
+      - name: Run tests for risk profiles
+        shell: bash {0}
+        run: bash testing/unit/run_report_test.sh testing/unit/risk_profile/risk_profile_test.py
+      - name: Run tests for reports
+        shell: bash {0}
+        run: bash testing/unit/run_report_test.sh testing/unit/report/report_test.py
+      - name: Archive HTML reports for modules
+        if: ${{ always() }}
+        run: sudo tar --exclude-vcs -czf html_reports.tgz testing/unit/report/output/
+      - name: Upload HTML reports
+        uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0
+        if: ${{ always() }}
+        with:
+          if-no-files-found: error
+          name: html-reports_${{ github.run_id }}
+          path: html_reports.tgz
 
   pylint:
     permissions: {}
@@ -98,7 +129,7 @@ jobs:
     - name: Install Node
       uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       with:
-        node-version: 18.18.0
+        node-version: 18.19.0
     - name: Install Chromium Browser
       run: sudo apt install chromium-browser
     - name: Install dependencies
@@ -121,7 +152,7 @@ jobs:
     - name: Install Node
       uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
       with:
-        node-version: 18.18.0
+        node-version: 18.19.0
     - name: Install dependencies
       run: npm install && npm ci
       working-directory: ./modules/ui

README.md

@@ -4,84 +4,93 @@
 [![CodeQL](https://github.com/google/testrun/actions/workflows/github-code-scanning/codeql/badge.svg?branch=main)](https://github.com/google/testrun/actions/workflows/github-code-scanning/codeql)
 [![Testrun test suite](https://github.com/google/testrun/actions/workflows/testing.yml/badge.svg?branch=main&event=push)](https://github.com/google/testrun/actions/workflows/testing.yml)
 
-## Introduction :wave:
-Testrun automates specific test cases to verify network and security functionality in IoT devices. It is an open source tool which allows manufacturers of IP capable devices to test their devices for the purposes of Device Qualification within the BOS program.
+# Introduction :wave:
 
-## Motivation :bulb:
-Without tools like Testrun, test labs and engineers may need to  maintain a large and complex network coupled with dynamic configuration files and constant software updates. The major issues which can and should be solved are:
- 1) The complexity of managing a testing network
- 2) The time required to perform testing of network functionality
- 3) The accuracy and consistency of testing network functionality
+Testrun automates specific test cases to verify network and security functionality in IoT devices. It's an open-source tool that manufacturers use to test their IP-capable devices for the purpose of device qualification within Google's Building Operating System  (BOS) program.
 
-## How it works :triangular_ruler:
-Testrun creates an isolated and controlled network environment on a linux machine. This removes the necessity for complex hardware, advanced knowledge and networking experience whilst enabling test engineers to validate device behaviour against Google’s Building Operating System requirements.
+# Motivation :bulb:
 
-Two modes are supported by Testrun:
+Test labs and engineers often need to maintain a large and complex network coupled with dynamic configuration files and constant software updates. Testrun helps address major issues like:
 
-<details>
-  <summary>
-    <strong>Automated testing</strong>
-  </summary>
+-  The complexity of managing a testing network
+-  The time required to perform testing of network functionality
+-  The accuracy and consistency of testing network functionality
 
-Once the device has become operational (steady state), automated testing of the DUT (device under test) will begin. Containerized test modules will then execute against the device, one module at a time. Once all test modules have been executed, a report will be produced - presenting the results.
-</details>
+# How it works :triangular_ruler:
 
-<details>
+Testrun creates an isolated and controlled network environment on a Linux machine. This removes the necessity for complex hardware, advanced knowledge, and networking experience while enabling test engineers to validate device behavior against Google's BOS requirements.
 
-  <summary>
-    <strong>Lab network</strong>
-  </summary>
+Testrun supports two modes: automated testing and lab network.
 
-When manual testing or configuration changes are required, Testrun will provide the network and some tools to assist an engineer performing the additional testing. This reduces the need to maintain a separate but identical lab network. Testrun will take care of packet captures and logs for each network service for further debugging.
+## Automated testing
 
-</details>
+Automated testing of the device under test (DUT) begins once the device is operational (steady state). Containerized test modules execute against the device one module at a time. Testrun produces a report with the results after all modules are executed.
 
-## Minimum requirements :computer:
-### Hardware
- - PC running Ubuntu LTS 20.04, 22.04 or 24.04 (laptop or desktop)
- - 2x USB ethernet adapter (One may be built in ethernet)
- - Internet connection
-### Software
-- Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
-### Device under test (DUT)
- - DHCP client - The device must be able to obtain an IP address via DHCP
+## Lab network
 
-## Get started ▶️
-Once you have met the hardware and software requirements, you can get started with Testrun by following the [Get started guide](docs/get_started.md). Further docs are available in the [docs directory](docs)
+Testrun provides the network and assistive tools for engineers when manual testing or configuration changes are required, reducing the need to maintain a separate but identical lab network. Testrun handles packet captures and logs for each network service for further debugging.
 
-## Roadmap :chart_with_upwards_trend:
-Testrun will constantly evolve to further support end-users by automating device network behaviour against industry standards. For further information on upcoming features, check out the [Roadmap](docs/roadmap.pdf).
+# Minimum requirements :computer:
 
-## Accessibility :busts_in_silhouette:
-We are proud of our tool and strive to provide an enjoyable experience for all of our users. Testrun goes through rigorous accessibility testing at each release. You can read more about [Google and Accessibility here](https://www.google.co.uk/accessibility). You are welcome to submit a new issue and provide feedback on our implementations. To find out how Testrun implements accessibility features, you can view a [short video here](docs/ui/accessibility.mp4).
+## Hardware
 
-## Issue reporting :triangular_flag_on_post:
-If the application has come across a problem at any point during setup or use, please raise an issue under the [issues tab](https://github.com/google/testrun/issues). Issue templates exist for both bug reports and feature requests. If neither of these are appropriate for your issue, raise a blank issue instead.
+-  PC running Ubuntu LTS 20.04, 22.04, or 24.04 (laptop or desktop)
+-  2x USB Ethernet adapter (one may be built-in Ethernet)
+-  Internet connection
 
-## Contributing :keyboard:
-The contributing requirements can be found in [CONTRIBUTING.md](CONTRIBUTING.md). In short, checkout the [Google CLA](https://cla.developers.google.com/) site to get started. After that, check out our [developer documentation](docs/dev/README.md).
+## Software
 
-## FAQ :raising_hand:
-1) I have an issue whilst installing/upgrading Testrun, what do I do?
+Testrun requires Docker. Refer to the [installation guide](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository) for more information.
 
-  Sometimes, issues may arise when installing or upgrading Testrun - this may happen due to one of many reasons due to the nature of the application. However, most of the time, it can be resolved by following a full Testrun re-install by using these commands:
-   - ```sudo docker system prune -a```
-   - ```sudo apt install ./testrun-*.deb```
+## Device under test (DUT)
 
-2) What device networking functionality is validated by Testrun?
+The DUT must be able to obtain an IP address via DHCP.
 
-  Best practices and requirements for IoT devices are constantly changing due to technological advances and discovery of vulnerabilities. 
-  The current expectations for IoT devices on Google deployments can be found in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements).
-  Testrun aims to automate as much of the Application Security Requirements as possible.
+# Get started :arrow_forward:
 
-3) What services are provided on the virtual network?
+Once you meet the hardware and software requirements, follow the Testrun [Get started guide](/docs/get_started.md). Additional guidance is available in the [docs directory](/docs).
 
-  The following are network services that are containerized and accessible to the device under test though are likely to change over time:
- - DHCP in failover configuration with internet connectivity
- - IPv6 SLAAC
- - DNS
- - NTPv4
+# Roadmap :chart_with_upwards_trend:
 
-4) Can I run Testrun on a virtual machine?
+Testrun continually evolves to further support end users by automating device network behavior against industry standards. For information on upcoming features, check out the [Roadmap](/docs/roadmap.pdf).
 
-  Testrun can be virtualized if the 2x ethernet adapters are passed through to a VirtualBox VM as a USB device rather than managed network adapters. You can view the guide to working on a [virtual machine here](docs/virtual_machine.md).
+# Accessibility :busts_in_silhouette:
+
+We're proud of our tool and strive to provide an enjoyable experience for everyone. Testrun goes through rigorous accessibility testing at each release. Download the [Testrun: Accessible features](https://github.com/google/testrun/raw/refs/heads/main/docs/ui/accessibility.mp4) video to learn more.You're welcome to [submit a new issue](https://github.com/google/testrun/issues) and provide feedback on our implementations. To learn more about Google's [Belonging initiative](https://www.google.co.uk/accessibility) and their approach to accessibility, visit their site.
+
+# Issue reporting :triangular_flag_on_post:
+
+If you encounter a problem during setup or use, raise an issue under the [Issues tab](https://github.com/google/testrun/issues). Issue templates exist for both bug reports and feature requests. If neither of these apply, raise a blank issue instead.
+
+# Contributing :keyboard:
+
+We strongly encourage contributions from the community. Review the requirements on the  ["How to Contribute" page](CONTRIBUTING.md), then follow the [developer guidelines](/docs/dev/README.md). 
+
+# FAQ :raising_hand:
+
+#### 1. What should I do if I have an issue while installing or upgrading Testrun?
+
+ You can resolve most issues by reinstalling Testrun using these commands:
+- `sudo docker system prune -a`
+- `sudo apt install ./testrun-*.deb`
+
+If this doesn't resolve the problem, [raise an issue](https://github.com/google/testrun/issues).
+
+#### 2. What device networking functionality does Testrun validate?
+
+Best practices and requirements for IoT devices change often due to technological advances and discovery of vulnerabilities. You can find the current expectations for IoT devices on Google deployments in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements). Testrun aims to automate as much of the Application Security Requirements as possible.
+
+#### 3. What services are provided on the virtual network?
+
+The following network services are containerized and accessible to the DUT:
+
+-  DHCP in failover configuration with internet connectivity
+-  IPv6 SLAAC
+-  DNS
+-  NTPv4
+
+Note that this list is likely to change over time.
+
+#### 4. Can I run Testrun on a virtual machine?
+
+Testrun can be virtualized if the 2x Ethernet adapters are passed through to a VirtualBox VM as a USB device rather than managed network adapters. Visit the [virtual machine guide](/docs/virtual_machine.md) for additional details.

cmd/build

@@ -60,11 +60,10 @@ fi
 
 # Build network modules
 echo Building network modules
-mkdir -p build/network
 for dir in modules/network/* ; do
   module=$(basename $dir)
   echo Building network module $module...
-  if docker build -f modules/network/$module/$module.Dockerfile -t test-run/$module . ; then
+  if docker build -f modules/network/$module/$module.Dockerfile -t testrun/$module . ; then
     echo Successfully built container for network $module
   else
     echo An error occured whilst building container for network module $module
@@ -74,11 +73,10 @@ done
 
 # Build validators
 echo Building network validators
-mkdir -p build/devices
 for dir in modules/devices/* ; do
   module=$(basename $dir)
   echo Building validator module $module...
-  if docker build -f modules/devices/$module/$module.Dockerfile -t test-run/$module . ; then
+  if docker build -f modules/devices/$module/$module.Dockerfile -t testrun/$module . ; then
     echo Successfully built container for device module $module
   else
     echo An error occured whilst building container for device module $module
@@ -88,11 +86,10 @@ done
 
 # Build test modules
 echo Building test modules
-mkdir -p build/test
 for dir in modules/test/* ; do
   module=$(basename $dir)
   echo Building test module $module...
-  if docker build -f modules/test/$module/$module.Dockerfile -t test-run/$module-test . ; then
+  if docker build -f modules/test/$module/$module.Dockerfile -t testrun/$module-test . ; then
     echo Successfully built container for test module $module
   else
     echo An error occured whilst building container for test module $module

cmd/install

@@ -68,15 +68,13 @@ deactivate
 cmd/build
 
 # Create local folders
-mkdir -p local/devices
-mkdir -p local/root_certs
-mkdir -p local/risk_profiles
+mkdir -p local/{devices,root_certs,risk_profiles}
 
 # Set file permissions on local
 # This does not work on GitHub actions
 if logname ; then
   USER_NAME=$(logname)
-  sudo chown -R "$USER_NAME" local
+  sudo chown -R "$USER_NAME" local || true
 fi
 
 echo Finished installing Testrun

cmd/prune

@@ -25,17 +25,16 @@ fi
 
 # Remove docker images
 echo Removing docker images
-docker_images=$(sudo docker images --filter=reference="test-run/*" -q)
+docker_images=$(sudo docker images --filter=reference="testrun/*" -q)
 
 if [ -z "$docker_images" ]; then
   echo No docker images to delete
 else
-  sudo docker rmi $docker_images > /dev/null
+  sudo docker rmi $docker_images
 fi
 
 # Remove docker networks
 echo Removing docker networks
-sudo docker network rm endev0 > /dev/null
-# Private network not used, add cleanup
-# back in if/when implemented
-#sudo docker network rm tr-private-net > /dev/null
+sudo docker network rm endev0 || true
+
+echo Successfully pruned Testrun resources

docs/README.md

@@ -1,21 +1,19 @@
 <img width="200" alt="Testrun logo" src="https://user-images.githubusercontent.com/7399056/221927867-4190a4e8-a571-4e40-9c2b-65780ad9264c.png" alt="Testrun">
 
+# Contents
 
-## Contents
+-  [Get started](/docs/get_started.md)
+    -  [Run on a virtual machine](/docs/virtual_machine.md)
+-  [Network](/docs/network/README.md)
+    -  [Network addresses](/docs/network/addresses.md)
+    -  [Add a new network service](/docs/network/add_new_service.md)
+-  [Testing](/docs/test/README.md)
+    -  [Test modules](/docs/test/modules.md)
+    -  [Test results](/docs/test/statuses.md)
+-  [Developer guidelines](/docs/dev/README.md)
+-  [Accessibility](/docs/ui/accessibility.md)
+-  [Roadmap](/docs/roadmap.pdf)
 
- - [Get Started](get_started.md)
- - [Network](network/README.md)
-   - [Network Overview](network/README.md)
-   - [How to identify network interfaces](network/identify_interfaces.md)
-   - [Addresses](network/addresses.md)
-   - [Add a new network service](network/add_new_service.md)
- - [Testing](test/README.md)
-   - [Test modules](test/modules.md)
-   - [Test statuses](test/statuses.md)
- - [Development](dev/README.md)
-   - [Running on a virtual machine](virtual_machine.md)
-   - [Accessibility](ui/accessibility.mp4)
-   - [Roadmap](roadmap.pdf)
+# Something missing?
 
-## Something missing?
-If you feel there is some documentation that you would find useful, or have found an issue with existing documentation, please raise an issue on GitHub by navigating [here](https://github.com/google/testrun/issues/new/choose)
+To request additional documentation or report an issue with existing resources, visit [the Issues tab](https://github.com/google/testrun/issues/new/choose).