From 555acb89376d46afac5773990cad20f8f91aa6b0 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Sun, 28 Jan 2024 10:43:00 -0800 Subject: [PATCH] Introduce dependency test suite The plan is to build out this test suite to test against the AWS CLI's dependencies to help facilitate dependency upgrades. To start, this test suite contains the following new test cases to better monitor the overall dependency closure of the awscli package: * Assert expected packages in runtime closure. This will alert us if a dependency introduces a new transitive depenency to the AWS CLI closure. * Assert expected unbounded dependencies in runtime closure. Specifically these are dependencies that do not have a version ceiling. This will alert us if a new unbounded dependency is introduced into the AWS CLI runtime dependency closure. See additional implementation notes below: * These tests were broken into a separate test suite (i.e. instead of adding them to the unit and functional test suite) to allow more granularity when running them. Specifically, it is useful for: 1. Avoiding the main unit and functional CI test suite from failing if a dependency changes from underneath of us (e.g. a new build dependency is added that we cannot control). 2. For individuals that package the awscli, they generally will not want to run this test suite as it is fairly specific to how pip installs dependencies. * To determine the runtime dependency closure, the Package and DependencyClosure utilities traverse the dist-info METADATA files of the packages installed in the current site packages to build the runtime graph. This approach was chosen because: 1. Since pip already installed the package, this logic avoids having to reconstruct the logic of how pip decides to resolve dependencies to figure out how to traverse the runtime graph. Any custom logic may deviate from how pip behaves which is what most users will be using to install the awscli as a Python package 2. It's faster. The runtime closure test cases do not require downloading or installing any additional packages. --- .github/workflows/run-dep-tests.yml | 27 +++++ requirements-dev-lock.txt | 14 ++- requirements-dev.txt | 3 + scripts/ci/run-dep-tests | 35 +++++++ tests/dependencies/__init__.py | 12 +++ tests/dependencies/test_closure.py | 146 ++++++++++++++++++++++++++++ 6 files changed, 229 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/run-dep-tests.yml create mode 100755 scripts/ci/run-dep-tests create mode 100644 tests/dependencies/__init__.py create mode 100644 tests/dependencies/test_closure.py diff --git a/.github/workflows/run-dep-tests.yml b/.github/workflows/run-dep-tests.yml new file mode 100644 index 000000000000..3ef275a63fe9 --- /dev/null +++ b/.github/workflows/run-dep-tests.yml @@ -0,0 +1,27 @@ +name: Run dependency tests + +on: + push: + pull_request: + branches-ignore: [ master ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + os: [ubuntu-latest, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: python scripts/ci/install + - name: Run tests + run: python scripts/ci/run-dep-tests diff --git a/requirements-dev-lock.txt b/requirements-dev-lock.txt index 05bb6ee0baa1..b1d4d0471b97 100644 --- a/requirements-dev-lock.txt +++ b/requirements-dev-lock.txt @@ -83,18 +83,16 @@ iniconfig==1.1.1 \ --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 # via pytest -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via pytest +packaging==23.2 \ + --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 + # via + # -r requirements-dev.txt + # pytest pluggy==1.0.0 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 # via pytest -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via packaging pytest==7.4.0 \ --hash=sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32 \ --hash=sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a diff --git a/requirements-dev.txt b/requirements-dev.txt index 5acb4e9a3602..6a3aa55ea639 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,3 +7,6 @@ pytest==7.4.0 pytest-cov==4.1.0 atomicwrites>=1.0 # Windows requirement colorama>0.3.0 # Windows requirement + +# Dependency test specific deps +packaging==23.2 diff --git a/scripts/ci/run-dep-tests b/scripts/ci/run-dep-tests new file mode 100755 index 000000000000..0cc0068e9eb0 --- /dev/null +++ b/scripts/ci/run-dep-tests @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# Don't run tests from the root repo dir. +# We want to ensure we're importing from the installed +# binary package not from the CWD. + +import os +import sys +from contextlib import contextmanager +from subprocess import check_call + +_dname = os.path.dirname + +REPO_ROOT = _dname(_dname(_dname(os.path.abspath(__file__)))) + + +@contextmanager +def cd(path): + """Change directory while inside context manager.""" + cwd = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(cwd) + + +def run(command): + env = os.environ.copy() + env['TESTS_REMOVE_REPO_ROOT_FROM_PATH'] = 'true' + return check_call(command, shell=True, env=env) + + +if __name__ == "__main__": + with cd(os.path.join(REPO_ROOT, "tests")): + run(f"{sys.executable} {REPO_ROOT}/scripts/ci/run-tests dependencies") diff --git a/tests/dependencies/__init__.py b/tests/dependencies/__init__.py new file mode 100644 index 000000000000..85c792b31b96 --- /dev/null +++ b/tests/dependencies/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/tests/dependencies/test_closure.py b/tests/dependencies/test_closure.py new file mode 100644 index 000000000000..529b7da553e6 --- /dev/null +++ b/tests/dependencies/test_closure.py @@ -0,0 +1,146 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import functools +import importlib.metadata +import json +from typing import Dict, Iterator, List, Tuple + +import pytest +from packaging.requirements import Requirement + +_NESTED_STR_DICT = Dict[str, "_NESTED_STR_DICT"] + + +@pytest.fixture() +def awscli_package(): + return Package(name="awscli") + + +class Package: + def __init__(self, name: str) -> None: + self.name = name + + @functools.cached_property + def runtime_dependencies(self) -> "DependencyClosure": + return self._get_runtime_closure() + + def _get_runtime_closure(self) -> "DependencyClosure": + closure = DependencyClosure() + for requirement in self._get_runtime_requirements(): + if self._requirement_applies_to_environment(requirement): + closure[requirement] = Package(name=requirement.name) + return closure + + def _get_runtime_requirements(self) -> List[Requirement]: + req_strings = importlib.metadata.distribution(self.name).requires + if req_strings is None: + return [] + return [Requirement(req_string) for req_string in req_strings] + + def _requirement_applies_to_environment( + self, requirement: Requirement + ) -> bool: + # Do not include any requirements defined as extras as currently + # our dependency closure does not use any extras + if requirement.extras: + return False + # Only include requirements where the markers apply to the current + # environment. + if requirement.marker and not requirement.marker.evaluate(): + return False + return True + + +class DependencyClosure: + def __init__(self) -> None: + self._req_to_package: Dict[Requirement, Package] = {} + + def __setitem__(self, key: Requirement, value: Package) -> None: + self._req_to_package[key] = value + + def __getitem__(self, key: Requirement) -> Package: + return self._req_to_package[key] + + def __delitem__(self, key: Requirement) -> None: + del self._req_to_package[key] + + def __iter__(self) -> Iterator[Requirement]: + return iter(self._req_to_package) + + def __len__(self) -> int: + return len(self._req_to_package) + + def walk(self) -> Iterator[Tuple[Requirement, Package]]: + for req, package in self._req_to_package.items(): + yield req, package + yield from package.runtime_dependencies.walk() + + def to_dict(self) -> _NESTED_STR_DICT: + reqs = {} + for req, package in self._req_to_package.items(): + reqs[str(req)] = package.runtime_dependencies.to_dict() + return reqs + + +class TestDependencyClosure: + def _is_bounded_version_requirement( + self, requirement: Requirement + ) -> bool: + for specifier in requirement.specifier: + if specifier.operator in ["==", "<=", "<"]: + return True + return False + + def _pformat_closure(self, closure: DependencyClosure) -> str: + return json.dumps(closure.to_dict(), sort_keys=True, indent=2) + + def test_expected_runtime_dependencies(self, awscli_package): + expected_dependencies = { + "botocore", + "colorama", + "docutils", + "jmespath", + "pyasn1", + "python-dateutil", + "PyYAML", + "rsa", + "s3transfer", + "six", + "urllib3", + } + actual_dependencies = set() + for _, package in awscli_package.runtime_dependencies.walk(): + actual_dependencies.add(package.name) + assert actual_dependencies == expected_dependencies, ( + f"Unexpected dependency found in runtime closure: " + f"{self._pformat_closure(awscli_package.runtime_dependencies)}" + ) + + def test_expected_unbounded_runtime_dependencies(self, awscli_package): + expected_unbounded_dependencies = { + "pyasn1", # Transitive dependency from rsa + "six", # Transitive dependency from python-dateutil + } + all_dependencies = set() + bounded_dependencies = set() + for req, package in awscli_package.runtime_dependencies.walk(): + all_dependencies.add(package.name) + if self._is_bounded_version_requirement(req): + bounded_dependencies.add(package.name) + actual_unbounded_dependencies = all_dependencies - bounded_dependencies + assert ( + actual_unbounded_dependencies == expected_unbounded_dependencies + ), ( + f"Unexpected unbounded dependency found in runtime closure: " + f"{self._pformat_closure(awscli_package.runtime_dependencies)}" + )