Release: 1.0.2 (#130)

* Set cancelled if stop button pressed * Add uninstall script * Re-add startup timeout in system.json.example * Serve report through API * Fix report URL * Increase timeout for tests * Remove tester3 * Resolve test cut off * Update artifact * Add timeout * Fix timezone in containers * Fix test cut off * Update version --------- Signed-off-by: J Boddey <[email protected]>
google · Oct 25, 2023 · 3389c2f · 3389c2f
1 parent e7e7365
commit 3389c2f
Show file tree

Hide file tree

Showing 28 changed files with 273 additions and 208 deletions.
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -21,6 +21,7 @@ jobs:
         run: cmd/package
       - name: Install Testrun
         shell: bash {0}
+        timeout-minutes: 10
         run: sudo dpkg -i testrun*.deb
       - name: Run baseline tests
         shell: bash {0}
@@ -30,7 +31,7 @@ jobs:
     name:  Tests
     runs-on: ubuntu-20.04
     needs: testrun_baseline
-    timeout-minutes: 45
+    timeout-minutes: 40
     steps:
       - name: Checkout source
         uses: actions/[email protected]
@@ -43,12 +44,13 @@ jobs:
       - name: Install Testrun
         shell: bash {0}
         run: sudo dpkg -i testrun*.deb
+        timeout-minutes: 10
       - name: Run tests
         shell: bash {0}
         run: testing/tests/test_tests
       - name: Archive runtime results
         if: ${{ always() }}
-        run: sudo tar --exclude-vcs -czf runtime.tgz /usr/local/testrun/runtime/
+        run: sudo tar --exclude-vcs -czf runtime.tgz /usr/local/testrun/runtime/ /usr/local/testrun/local/
       - name: Upload runtime results
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
@@ -73,6 +75,7 @@ jobs:
       - name: Install Testrun
         shell: bash {0}
         run: sudo dpkg -i testrun*.deb
+        timeout-minutes: 10
       - name: Run tests
         shell: bash {0}
         run: testing/api/test_api

diff --git a/README.md b/README.md
@@ -31,6 +31,8 @@ Testrun cannot automate everything, and so additional manual testing may be requ
  - Internet connection
 ### Software
 - Docker - installation guide: [https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository](https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository)
+### Device
+ - DHCP client - The device must be able to obtain an IP address via DHCP
 
 ## Get started ▶️
 Once you have met the hardware and software requirements, you can get started with Testrun by following the [Get started guide](docs/get_started.md).

diff --git a/bin/testrun b/bin/testrun
@@ -19,10 +19,6 @@ if [[ "$EUID" -ne 0 ]]; then
     exit 1
 fi
 
-# TODO: Obtain TESTRUNPATH from user environment variables
-# TESTRUNPATH="/home/boddey/Desktop/test-run"
-# cd $TESTRUNPATH
-
 # Ensure that /var/run/netns folder exists
 sudo mkdir -p /var/run/netns
 

diff --git a/cmd/package b/cmd/package
@@ -53,4 +53,4 @@ cp -r {framework,modules} $MAKE_SRC_DIR/usr/local/testrun
 dpkg-deb --build --root-owner-group make
 
 # Rename the .deb file
-mv make.deb testrun_1-0-1_amd64.deb
+mv make.deb testrun_1-0-2_amd64.deb
diff --git a/docs/configure_device.md b/docs/configure_device.md
diff --git a/docs/get_started.md b/docs/get_started.md
@@ -24,6 +24,11 @@ Ensure the following software is installed on your Ubuntu LTS PC:
    - Build Essential
    - Net Tools
 
+### Device
+Any device with an ethernet connection, and support for IPv4 DHCP can be tested.
+
+However, to achieve a compliant test outcome, your device must be configured correctly and implement the required security features. These standards are outlined in the [Application Security Requirements for IoT Devices](https://partner-security.withgoogle.com/docs/iot_requirements). but further detail is available in [documentation for each test module](/docs/test/modules.md).
+
 ## Installation
 
 1. Download the latest version of the Testrun installer from the [releases page](https://github.com/google/test-run/releases)
@@ -42,6 +47,8 @@ Ensure the following software is installed on your Ubuntu LTS PC:
    - Connect one USB Ethernet adapter to the internet source (e.g., router or switch) using an ethernet cable.
    - Connect the other USB Ethernet adapter directly to the IoT device you want to test using an ethernet cable.
 
+   **NOTE: The device under test should be powered off until prompted**
+
    **NOTE: Both adapters should be disabled in the host system (IPv4, IPv6 and general). You can do this by going to Settings > Network** 
 
 2. Start Testrun.
@@ -82,7 +89,9 @@ Start Testrun with the command `sudo testrun`
 
  - During testing, if you would like to stop Testrun, click 'Stop' next to the test name.
 
-11. On completion of the test sequence, a report will appear under the history icon. 
+11. Once the notification 'Waiting for Device' appears, power on the device under test.
+
+12. On completion of the test sequence, a report will appear under the history icon. 
 
     ![](/docs/ui/history_icon.png)
 
@@ -94,3 +103,6 @@ If you encounter any issues or need assistance, consider the following:
 - Verify that the network interfaces are connected correctly.
 - Check the configuration settings.
 - Refer to the Testrun documentation or ask for assistance in the issues page: https://github.com/google/testrun/issues
+
+# Uninstall
+To uninstall Testrun, use the built-in dpkg uninstall command to remove Testrun correctly. For Testrun, this would be:  ```sudo apt-get remove testrun```
diff --git a/docs/network/addresses.md b/docs/network/addresses.md
@@ -4,13 +4,13 @@ Each network service is configured with an IPv4 and IPv6 address. For IPv4 addre
 
 | Name                | Mac address          | IPv4 address | IPv6 address                 |
 |---------------------|----------------------|--------------|------------------------------|
-| Internet gateway    | 9a:02:57:1e:8f:01   | 10.10.10.1   | fd10:77be:4186::1            |
-| DHCP primary        | 9a:02:57:1e:8f:02   | 10.10.10.2   | fd10:77be:4186::2            |
-| DHCP secondary      | 9a:02:57:1e:8f:03   | 10.10.10.3   | fd10:77be:4186::3            |
-| DNS server          | 9a:02:57:1e:8f:04   | 10.10.10.4   | fd10:77be:4186::4            |
-| NTP server          | 9a:02:57:1e:8f:05   | 10.10.10.5   | fd10:77be:4186::5            |
-| Radius authenticator| 9a:02:57:1e:8f:07   | 10.10.10.7   | fd10:77be:4186::7            |
-| Active test module  | 9a:02:57:1e:8f:09   | 10.10.10.9   | fd10:77be:4186::9            |
+| Internet gateway    | 9a:02:57:1e:8f:01    | 10.10.10.1   | fd10:77be:4186::1            |
+| DHCP primary        | 9a:02:57:1e:8f:02    | 10.10.10.2   | fd10:77be:4186::2            |
+| DHCP secondary      | 9a:02:57:1e:8f:03    | 10.10.10.3   | fd10:77be:4186::3            |
+| DNS server          | 9a:02:57:1e:8f:04    | 10.10.10.4   | fd10:77be:4186::4            |
+| NTP server          | 9a:02:57:1e:8f:05    | 10.10.10.5   | fd10:77be:4186::5            |
+| Radius authenticator| 9a:02:57:1e:8f:07    | 10.10.10.7   | fd10:77be:4186::7            |
+| Active test module  | 9a:02:57:1e:8f:09    | 10.10.10.9   | fd10:77be:4186::9            |
 
 
 The default network range is 10.10.10.0/24 and devices will be assigned addresses in that range via DHCP. The range may change when requested by a test module. In which case, network services will be restarted and accessible on the new range, with the same final host ID. The default IPv6 network is fd10:77be:4186::/64 and addresses will be assigned to devices on the network using IPv6 SLAAC.

diff --git a/framework/python/src/api/api.py b/framework/python/src/api/api.py
@@ -13,9 +13,11 @@
 # limitations under the License.
 
 from fastapi import FastAPI, APIRouter, Response, Request, status
+from fastapi.responses import FileResponse
 from fastapi.middleware.cors import CORSMiddleware
 import json
 from json import JSONDecodeError
+import os
 import psutil
 import threading
 import uvicorn
@@ -29,6 +31,7 @@
 DEVICE_MANUFACTURER_KEY = "manufacturer"
 DEVICE_MODEL_KEY = "model"
 DEVICE_TEST_MODULES_KEY = "test_modules"
+DEVICES_PATH = "/usr/local/testrun/local/devices"
 
 class Api:
   """Provide REST endpoints to manage Testrun"""
@@ -50,7 +53,11 @@ def __init__(self, test_run):
     self._router.add_api_route("/system/stop", self.stop_test_run,
                                methods=["POST"])
     self._router.add_api_route("/system/status", self.get_status)
+
     self._router.add_api_route("/history", self.get_history)
+    self._router.add_api_route("/report/{device_name}/{timestamp}",
+                               self.get_report)
+
     self._router.add_api_route("/devices", self.get_devices)
     self._router.add_api_route("/device", self.save_device, methods=["POST"])
 
@@ -140,19 +147,24 @@ async def start_test_run(self, request: Request, response: Response):
     # Check if requested device is known in the device repository
     if device is None:
       response.status_code = status.HTTP_404_NOT_FOUND
-      return self._generate_msg(False,
-                                "A device with that MAC address could not be found")
+      return self._generate_msg(
+        False,
+        "A device with that MAC address could not be found")
 
     device.firmware = body_json["device"]["firmware"]
 
     # Check Testrun is able to start
     if self._test_run.get_net_orc().check_config() is False:
       response.status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
-      return self._generate_msg(False,"Configured interfaces are not ready for use. Ensure required interfaces are connected.")
+      return self._generate_msg(False,"Configured interfaces are not " +
+                                "ready for use. Ensure required interfaces " +
+                                "are connected.")
 
     self._test_run.get_session().reset()
     self._test_run.get_session().set_target_device(device)
-    LOGGER.info(f"Starting Testrun with device target {device.manufacturer} {device.model} with MAC address {device.mac_addr}")
+    LOGGER.info("Starting Testrun with device target " +
+                f"{device.manufacturer} {device.model} with " +
+                f"MAC address {device.mac_addr}")
 
     thread = threading.Thread(target=self._start_test_run,
                                         name="Testrun")
@@ -170,7 +182,10 @@ def _start_test_run(self):
 
   async def stop_test_run(self):
     LOGGER.debug("Received stop command. Stopping Testrun")
+
+    # TODO: Set status of 'Stopping'?
     self._test_run.stop()
+
     return self._generate_msg(True, "Testrun stopped")
 
   async def get_status(self):
@@ -218,6 +233,19 @@ async def save_device(self, request: Request, response: Response):
       response.status_code = status.HTTP_400_BAD_REQUEST
       return self._generate_msg(False, "Invalid JSON received")
 
+  async def get_report(self, response: Response,
+                       device_name, timestamp):
+
+    file_path = os.path.join(DEVICES_PATH, device_name, "reports",
+                             timestamp, "report.pdf")
+    LOGGER.debug(f"Received get report request for {device_name} / {timestamp}")
+    if os.path.isfile(file_path):
+      return FileResponse(file_path)
+    else:
+      LOGGER.info("Report could not be found, returning 404")
+      response.status_code = 404
+      return self._generate_msg(False, "Report could not be found")
+
   def _validate_device_json(self, json_obj):
 
     # Check all required properties are present
@@ -241,4 +269,4 @@ def _validate_device_json(self, json_obj):
       if char in disallowed_chars:
         return False
 
-    return True
+    return True
diff --git a/framework/python/src/common/device.py b/framework/python/src/common/device.py
@@ -38,7 +38,15 @@ def add_report(self, report):
   def get_reports(self):
     return self.reports
 
-  # TODO: Add ability to remove reports once test reports have been cleaned up
+  def remove_report(self, timestamp):
+
+    remove_report_target = None
+    for report in self.reports:
+      if report.get_started() == timestamp:
+        remove_report_target = report
+
+    if remove_report_target is not None:
+      self.reports.remove(remove_report_target)
 
   def to_dict(self):
     """Returns the device as a python dictionary. This is used for the

diff --git a/framework/python/src/common/session.py b/framework/python/src/common/session.py
@@ -22,7 +22,6 @@
 NETWORK_KEY = 'network'
 DEVICE_INTF_KEY = 'device_intf'
 INTERNET_INTF_KEY = 'internet_intf'
-RUNTIME_KEY = 'runtime'
 MONITOR_PERIOD_KEY = 'monitor_period'
 STARTUP_TIMEOUT_KEY = 'startup_timeout'
 LOG_LEVEL_KEY = 'log_level'
@@ -47,6 +46,11 @@ def __init__(self, config_file):
     self._config = self._get_default_config()
     self._load_config()
 
+    tz = util.run_command('cat /etc/timezone')
+    # TODO: Check if timezone is fetched successfully
+    self._timezone = tz[0]
+    LOGGER.info(f'System timezone is {self._timezone}')
+
   def start(self):
     self.reset()
     self._status = 'Waiting for Device'
@@ -70,7 +74,6 @@ def _get_default_config(self):
       'log_level': 'INFO',
       'startup_timeout': 60,
       'monitor_period': 30,
-      'runtime': 120,
       'max_device_reports': 5,
       'api_port': 8000
     }
@@ -98,9 +101,6 @@ def _load_config(self):
         self._config[NETWORK_KEY][INTERNET_INTF_KEY] = config_file_json.get(
           NETWORK_KEY, {}).get(INTERNET_INTF_KEY)
 
-      if RUNTIME_KEY in config_file_json:
-        self._config[RUNTIME_KEY] = config_file_json.get(RUNTIME_KEY)
-
       if STARTUP_TIMEOUT_KEY in config_file_json:
         self._config[STARTUP_TIMEOUT_KEY] = config_file_json.get(
           STARTUP_TIMEOUT_KEY)
@@ -126,9 +126,6 @@ def _save_config(self):
       f.write(json.dumps(self._config, indent=2))
     util.set_file_owner(owner=util.get_host_user(), path=self._config_file)
 
-  def get_runtime(self):
-    return self._config.get(RUNTIME_KEY)
-
   def get_log_level(self):
     return self._config.get(LOG_LEVEL_KEY)
 
@@ -241,3 +238,6 @@ def to_json(self):
     }
 
     return session_json
+
+  def get_timezone(self):
+    return self._timezone