diff --git a/.github/workflows/quality-checks.yaml b/.github/workflows/quality-checks.yaml index 32a3301..30de224 100644 --- a/.github/workflows/quality-checks.yaml +++ b/.github/workflows/quality-checks.yaml @@ -44,7 +44,7 @@ jobs: run: git config --global --add safe.directory /__w/amazon-mwaa-docker-images/amazon-mwaa-docker-images - name: Create the necessary Python virtual environments... - run: python3.11 ./create_venvs.py + run: python3.11 ./create_venvs.py --target production - name: Run quality checks... run: python3.11 ./quality-checks/run_all.py diff --git a/.gitignore b/.gitignore index 69024e2..f444a49 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ .venv venv +# Pyenv local overrides +.python-version + # Python cache files __pycache__/ *.py[cod] @@ -12,3 +15,6 @@ __pycache__/ # Build directories build +# Ignore dev requirements files. +images/airflow/2.*/requirements-dev.txt +images/airflow/3.*/requirements-dev.txt diff --git a/README.md b/README.md index 898e327..401169c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ To experiment with the image using a vanilla Docker setup, follow these steps: package, execute the following command: ``` -python3 create_venvs.py +python3 create_venvs.py --target ``` 3. Build the Airflow v2.9.2 Docker image diff --git a/create_venvs.py b/create_venvs.py index 0fc8885..e344285 100644 --- a/create_venvs.py +++ b/create_venvs.py @@ -15,6 +15,7 @@ import argparse import os +import re import shutil import subprocess import sys @@ -31,11 +32,12 @@ def verify_python_version(): sys.exit(1) -def create_venv(path: Path, recreate: bool = False): +def create_venv(path: Path, development_build: bool, recreate: bool = False): """ Create a venv in the given directory and optionally recreate it if it already exists. :param path: The path to create the venv in. + :param development_build: Is this a development build. :param recreate: Whether to recreate the venv if it already exists. """ venv_path = path / ".venv" @@ -55,19 +57,50 @@ def create_venv(path: Path, recreate: bool = False): pip_install(venv_path, "-U", "pip") print("") - requirements_path = str(path / "requirements.txt") + requirements_path = generate_requirements(path, development_build) print(f"> Install dependencies from {requirements_path}...") - pip_install(venv_path, "-r", requirements_path) + pip_install(venv_path, "-r", str(requirements_path)) print("") - print("> Install/Upgrade development tools: pydocstyle, pyright, ruff...") - pip_install(venv_path, "-U", "pydocstyle", "pyright", "ruff") + dev_tools = ["pydocstyle", "pyright", "ruff"] + print(f"> Install/Upgrade development tools: {dev_tools}...") + pip_install(venv_path, "-U", *dev_tools) print("") print(f">>> Finished creating a virtual environment under the path {venv_path}.") print("") print("") +def generate_requirements(path: Path, development_build: bool) -> Path: + """ + If the requirements.txt file at the path needs to be updated for local development, generate + a new requirements file. + + Return the path to the requirements file to be used. + + :param path: The path to the directory containing the requirements.txt file. + :param development_build: Is this a development build. + """ + requirements_path = path.joinpath("requirements.txt") + + if not development_build: + print("> Production build selected. Using default requirements.") + return requirements_path + + if not re.search(r"images\/airflow\/[2-3]\.[0-9]+\.[0-9]+$", str(path.resolve())): + print(f"> No need to create dev requirements for {path.resolve()}. Using default.") + return requirements_path + + with open(requirements_path.resolve(), 'r') as file: + # psycopg2-binary is meant for development and removes the requirement to install pg_config + filedata = re.sub(r"\bpsycopg2\b", "psycopg2-binary", file.read()) + + dev_requirements_path = path.joinpath('requirements-dev.txt') + print(f"> Creating {dev_requirements_path} from {requirements_path}") + with open(dev_requirements_path.resolve(), 'w') as file: + file.write(filedata) + + return dev_requirements_path def pip_install(venv_dir: Path, *args: str): """ @@ -91,6 +124,12 @@ def main(): "--recreate", action="store_true", help="Recreate the venv if it exists" ) + development_target_choice = "development" + build_targets = [development_target_choice, "production"] + parser.add_argument( + "--target", choices=build_targets, required=True, help="Sets the build target" + ) + # Parse the arguments args = parser.parse_args() @@ -101,7 +140,11 @@ def main(): ] # Include main project dir and each image dir for dir_path in project_dirs: if dir_path.is_dir() and (dir_path / "requirements.txt").exists(): - create_venv(dir_path, recreate=args.recreate) + create_venv( + dir_path, + development_build=args.target == development_target_choice, + recreate=args.recreate + ) if __name__ == "__main__":