diff --git a/.github/workflows/analyses-snapshot-test.yaml b/.github/workflows/analyses-snapshot-test.yaml index 09539d873e9..fffdd6b667d 100644 --- a/.github/workflows/analyses-snapshot-test.yaml +++ b/.github/workflows/analyses-snapshot-test.yaml @@ -45,12 +45,13 @@ jobs: timeout-minutes: 15 runs-on: ubuntu-latest env: + BASE_IMAGE_NAME: opentrons-python-base:3.10 ANALYSIS_REF: ${{ github.event.inputs.ANALYSIS_REF || github.head_ref || 'edge' }} SNAPSHOT_REF: ${{ github.event.inputs.SNAPSHOT_REF || github.head_ref || 'edge' }} # If we're running because of workflow_dispatch, use the user input to decide # whether to open a PR on failure. Otherwise, there is no user input, # so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr' - OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.events.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }} + OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }} PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'not a pr'}} steps: - name: Checkout Repository @@ -71,9 +72,24 @@ jobs: echo "Analyses snapshots match ${{ env.PR_TARGET_BRANCH }} snapshots." fi - - name: Docker Build + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build base image + id: build_base_image + uses: docker/build-push-action@v6 + with: + context: analyses-snapshot-testing/citools + file: analyses-snapshot-testing/citools/Dockerfile.base + push: false + load: true + tags: ${{ env.BASE_IMAGE_NAME }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build analysis image working-directory: analyses-snapshot-testing - run: make build-opentrons-analysis + run: make build-opentrons-analysis BASE_IMAGE_NAME=${{ env.BASE_IMAGE_NAME }} ANALYSIS_REF=${{ env.ANALYSIS_REF }} CACHEBUST=${{ github.run_number }} - name: Set up Python 3.13 uses: actions/setup-python@v5 @@ -112,8 +128,8 @@ jobs: commit-message: 'fix(analyses-snapshot-testing): heal analyses snapshots' title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots' body: 'This PR was requested on the PR https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF}}' - base: ${{ env.SNAPSHOT_REF}} + branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}' + base: ${{ env.SNAPSHOT_REF }} - name: Comment on feature PR if: always() && steps.create_pull_request.outcome == 'success' && github.event_name == 'pull_request' @@ -135,5 +151,5 @@ jobs: commit-message: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots' title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots' body: 'The ${{ env.ANALYSIS_REF }} overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.' - branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF}}' - base: ${{ env.SNAPSHOT_REF}} + branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}' + base: ${{ env.SNAPSHOT_REF }} diff --git a/analyses-snapshot-testing/Makefile b/analyses-snapshot-testing/Makefile index 13c4e603f3c..de5e0381131 100644 --- a/analyses-snapshot-testing/Makefile +++ b/analyses-snapshot-testing/Makefile @@ -1,38 +1,56 @@ +BASE_IMAGE_NAME ?= opentrons-python-base:3.10 +CACHEBUST ?= $(shell date +%s) +ANALYSIS_REF ?= edge +PROTOCOL_NAMES ?= all +OVERRIDE_PROTOCOL_NAMES ?= all +OPENTRONS_VERSION ?= edge + +export OPENTRONS_VERSION # used for server +export ANALYSIS_REF # used for analysis and snapshot test +export PROTOCOL_NAMES # used for the snapshot test +export OVERRIDE_PROTOCOL_NAMES # used for the snapshot test + +ifeq ($(CI), true) + PYTHON=python +else + PYTHON=pyenv exec python +endif + .PHONY: black black: - python -m pipenv run python -m black . + $(PYTHON) -m pipenv run python -m black . .PHONY: black-check black-check: - python -m pipenv run python -m black . --check + $(PYTHON) -m pipenv run python -m black . --check .PHONY: ruff ruff: - python -m pipenv run python -m ruff check . --fix + $(PYTHON) -m pipenv run python -m ruff check . --fix .PHONY: ruff-check ruff-check: - python -m pipenv run python -m ruff check . + $(PYTHON) -m pipenv run python -m ruff check . .PHONY: mypy mypy: - python -m pipenv run python -m mypy automation tests citools + $(PYTHON) -m pipenv run python -m mypy automation tests citools .PHONY: lint lint: black-check ruff-check mypy .PHONY: format format: - @echo runnning black + @echo "Running black" $(MAKE) black - @echo running ruff + @echo "Running ruff" $(MAKE) ruff - @echo formatting the readme with yarn prettier + @echo "Formatting the readme with yarn prettier" $(MAKE) format-readme .PHONY: test-ci test-ci: - python -m pipenv run python -m pytest -m "emulated_alpha" + $(PYTHON) -m pipenv run python -m pytest -m "emulated_alpha" .PHONY: test-protocol-analysis test-protocol-analysis: @@ -40,66 +58,64 @@ test-protocol-analysis: .PHONY: setup setup: install-pipenv - python -m pipenv install + $(PYTHON) -m pipenv install .PHONY: teardown teardown: - python -m pipenv --rm + $(PYTHON) -m pipenv --rm .PHONY: format-readme format-readme: - yarn prettier --ignore-path .eslintignore --write analyses-snapshot-testing/**/*.md + yarn prettier --ignore-path .eslintignore --write analyses-snapshot-testing/**/*.md .github/workflows/analyses-snapshot-test.yaml .PHONY: install-pipenv install-pipenv: - python -m pip install -U pipenv - -ANALYSIS_REF ?= edge -PROTOCOL_NAMES ?= all -OVERRIDE_PROTOCOL_NAMES ?= all - -export ANALYSIS_REF -export PROTOCOL_NAMES -export OVERRIDE_PROTOCOL_NAMES + $(PYTHON) -m pip install -U pipenv .PHONY: snapshot-test snapshot-test: @echo "ANALYSIS_REF is $(ANALYSIS_REF)" @echo "PROTOCOL_NAMES is $(PROTOCOL_NAMES)" @echo "OVERRIDE_PROTOCOL_NAMES is $(OVERRIDE_PROTOCOL_NAMES)" - python -m pipenv run pytest -k analyses_snapshot_test -vv + $(PYTHON) -m pipenv run pytest -k analyses_snapshot_test -vv .PHONY: snapshot-test-update snapshot-test-update: @echo "ANALYSIS_REF is $(ANALYSIS_REF)" @echo "PROTOCOL_NAMES is $(PROTOCOL_NAMES)" @echo "OVERRIDE_PROTOCOL_NAMES is $(OVERRIDE_PROTOCOL_NAMES)" - python -m pipenv run pytest -k analyses_snapshot_test --snapshot-update + $(PYTHON) -m pipenv run pytest -k analyses_snapshot_test --snapshot-update -CACHEBUST := $(shell date +%s) +.PHONY: build-base-image +build-base-image: + @echo "Building the base image $(BASE_IMAGE_NAME)" + docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) -f citools/Dockerfile.base -t $(BASE_IMAGE_NAME) citools/. .PHONY: build-opentrons-analysis build-opentrons-analysis: @echo "Building docker image for $(ANALYSIS_REF)" @echo "The image will be named opentrons-analysis:$(ANALYSIS_REF)" @echo "If you want to build a different version, run 'make build-opentrons-analysis ANALYSIS_REF='" - @echo "Cache is always busted to ensure latest version of the code is used" - docker build --build-arg ANALYSIS_REF=$(ANALYSIS_REF) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-analysis:$(ANALYSIS_REF) citools/. + docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) --build-arg ANALYSIS_REF=$(ANALYSIS_REF) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-analysis:$(ANALYSIS_REF) -f citools/Dockerfile.analyze citools/. + +.PHONY: local-build +local-build: + @echo "Building docker image for your local opentrons code" + @echo "The image will be named opentrons-analysis:local" + @echo "For a fresh build, run 'make local-build NO_CACHE=1'" + docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) $(BUILD_FLAGS) -t opentrons-analysis:local -f citools/Dockerfile.local .. || true + @echo "Build complete" .PHONY: generate-protocols generate-protocols: - python -m pipenv run python -m automation.data.protocol_registry - - -OPENTRONS_VERSION ?= edge -export OPENTRONS_VERSION + $(PYTHON) -m pipenv run python -m automation.data.protocol_registry .PHONY: build-rs build-rs: @echo "Building docker image for opentrons-robot-server:$(OPENTRONS_VERSION)" @echo "Cache is always busted to ensure latest version of the code is used" @echo "If you want to build a different version, run 'make build-rs OPENTRONS_VERSION=chore_release-8.0.0'" - docker build --build-arg OPENTRONS_VERSION=$(OPENTRONS_VERSION) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-robot-server:$(OPENTRONS_VERSION) -f citools/Dockerfile.server . + docker build --build-arg BASE_IMAGE_NAME=$(BASE_IMAGE_NAME) --build-arg OPENTRONS_VERSION=$(OPENTRONS_VERSION) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-robot-server:$(OPENTRONS_VERSION) -f citools/Dockerfile.server . .PHONY: run-flex run-flex: diff --git a/analyses-snapshot-testing/README.md b/analyses-snapshot-testing/README.md index 51a8e194ca1..78423b8447f 100644 --- a/analyses-snapshot-testing/README.md +++ b/analyses-snapshot-testing/README.md @@ -18,7 +18,10 @@ > This ALWAYS gets the remote code pushed to Opentrons/opentrons for the specified ANALYSIS_REF -`make build-opentrons-analysis ANALYSIS_REF=chore_release-8.0.0` +- build the base image + - `make build-base-image` +- build the opentrons-analysis image + - `make build-opentrons-analysis ANALYSIS_REF=release` ## Running the tests locally @@ -51,10 +54,28 @@ ```shell cd analyses-snapshot-testing \ -&& make build-rs OPENTRONS_VERSION=chore_release-8.0.0 \ -&& make run-rs OPENTRONS_VERSION=chore_release-8.0.0` +&& make build-base-image \ +&& make build-rs OPENTRONS_VERSION=release \ +&& make run-rs OPENTRONS_VERSION=release` ``` ### Default OPENTRONS_VERSION=edge in the Makefile so you can omit it if you want latest edge -`cd analyses-snapshot-testing && make build-rs && make run-rs` +```shell +cd analyses-snapshot-testing \ +&& make build-base-image \ +&& make build-rs \ +&& make run-rs +``` + +## Running the Analyses Battery against your local code + +> This copies in your local code to the container and runs the analyses battery against it. + +1. `make build-base-image` +1. `make build-local` +1. `make local-snapshot-test` + +You have the option to specify one or many protocols to run the analyses on. This is also described above [Running the tests against specific protocols](#running-the-tests-against-specific-protocols) + +- `make local-snapshot-test PROTOCOL_NAMES=Flex_S_v2_19_Illumina_DNA_PCR_Free OVERRIDE_PROTOCOL_NAMES=none` diff --git a/analyses-snapshot-testing/citools/Dockerfile b/analyses-snapshot-testing/citools/Dockerfile.analyze similarity index 75% rename from analyses-snapshot-testing/citools/Dockerfile rename to analyses-snapshot-testing/citools/Dockerfile.analyze index 123b7636652..1b85981cdaf 100644 --- a/analyses-snapshot-testing/citools/Dockerfile +++ b/analyses-snapshot-testing/citools/Dockerfile.analyze @@ -1,10 +1,6 @@ -# Use 3.10 just like the app does -FROM python:3.10-slim-bullseye +ARG BASE_IMAGE_NAME=opentrons-python-base:3.10 -# Update packages and install git -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y git libsystemd-dev +FROM ${BASE_IMAGE_NAME} # Define build arguments ARG ANALYSIS_REF=edge diff --git a/analyses-snapshot-testing/citools/Dockerfile.base b/analyses-snapshot-testing/citools/Dockerfile.base new file mode 100644 index 00000000000..086987e671b --- /dev/null +++ b/analyses-snapshot-testing/citools/Dockerfile.base @@ -0,0 +1,7 @@ +# Use Python 3.10 as the base image +FROM python:3.10-slim-bullseye + +# Update packages and install dependencies +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y git libsystemd-dev build-essential pkg-config network-manager diff --git a/analyses-snapshot-testing/citools/Dockerfile.local b/analyses-snapshot-testing/citools/Dockerfile.local new file mode 100644 index 00000000000..2346b4680c2 --- /dev/null +++ b/analyses-snapshot-testing/citools/Dockerfile.local @@ -0,0 +1,19 @@ +ARG BASE_IMAGE_NAME=opentrons-python-base:3.10 + +FROM ${BASE_IMAGE_NAME} + +# Set the working directory in the container +WORKDIR /opentrons + +# Copy everything from the build context into the /opentrons directory +# root directory .dockerignore file is respected +COPY . /opentrons + +# Install required packages from the copied code +RUN python -m pip install -U ./shared-data/python +RUN python -m pip install -U ./hardware[flex] +RUN python -m pip install -U ./api +RUN python -m pip install -U pandas==1.4.3 + +# The default command to keep the container running +CMD ["tail", "-f", "/dev/null"] diff --git a/analyses-snapshot-testing/citools/Dockerfile.server b/analyses-snapshot-testing/citools/Dockerfile.server index 6d4d9edcda3..0c44c1e04f0 100644 --- a/analyses-snapshot-testing/citools/Dockerfile.server +++ b/analyses-snapshot-testing/citools/Dockerfile.server @@ -1,10 +1,6 @@ -# Use Python 3.10 as the base image -FROM python:3.10-slim-bullseye +ARG BASE_IMAGE_NAME=opentrons-python-base:3.10 -# Update packages and install dependencies -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y git libsystemd-dev build-essential pkg-config network-manager +FROM ${BASE_IMAGE_NAME} # Define build arguments ARG OPENTRONS_VERSION=edge diff --git a/analyses-snapshot-testing/citools/generate_analyses.py b/analyses-snapshot-testing/citools/generate_analyses.py index 52aba70363b..7d550b47776 100644 --- a/analyses-snapshot-testing/citools/generate_analyses.py +++ b/analyses-snapshot-testing/citools/generate_analyses.py @@ -24,7 +24,7 @@ HOST_RESULTS: Path = Path(Path(__file__).parent.parent, "analysis_results") ANALYSIS_SUFFIX: str = "analysis.json" ANALYSIS_TIMEOUT_SECONDS: int = 30 -ANALYSIS_CONTAINER_INSTANCES: int = 5 +MAX_ANALYSIS_CONTAINER_INSTANCES: int = 5 console = Console() @@ -241,6 +241,12 @@ def analyze_against_image(tag: str, protocols: List[TargetProtocol], num_contain return protocols +def get_container_instances(protocol_len: int) -> int: + # Scaling linearly with the number of protocols + instances = max(1, min(MAX_ANALYSIS_CONTAINER_INSTANCES, protocol_len // 10)) + return instances + + def generate_analyses_from_test(tag: str, protocols: List[Protocol]) -> None: """Generate analyses from the tests.""" start_time = time.time() @@ -260,6 +266,7 @@ def generate_analyses_from_test(tag: str, protocols: List[Protocol]) -> None: protocol_custom_labware_paths_in_container(test_protocol), ) ) - analyze_against_image(tag, protocols_to_process, ANALYSIS_CONTAINER_INSTANCES) + instance_count = get_container_instances(len(protocols_to_process)) + analyze_against_image(tag, protocols_to_process, instance_count) end_time = time.time() console.print(f"Clock time to generate analyses: {end_time - start_time:.2f} seconds.")