From a05d03d5e36bd534e565076ad1756f352782bc3f Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Wed, 6 Sep 2023 14:54:51 -0400 Subject: [PATCH 1/5] Consolidate image and server setup in multiple checks --- .github/actions/awx_ci_setup/action.yml | 36 +++++++++++++++++++++ .github/actions/run_awx_devel/action.yml | 20 ------------ .github/workflows/ci.yml | 25 ++++++++++++-- .github/workflows/e2e_test.yml | 5 +++ Makefile | 13 +------- tools/docker-compose/ansible/smoke-test.yml | 17 ---------- 6 files changed, 65 insertions(+), 51 deletions(-) create mode 100644 .github/actions/awx_ci_setup/action.yml diff --git a/.github/actions/awx_ci_setup/action.yml b/.github/actions/awx_ci_setup/action.yml new file mode 100644 index 000000000000..552d3e61dda7 --- /dev/null +++ b/.github/actions/awx_ci_setup/action.yml @@ -0,0 +1,36 @@ +# This currently *always* uses the "warm build cache" image +# We should do something to allow forcing a rebuild, probably by looking for +# some string in the commit message or something. + +name: Setup images for AWX +description: Builds new awx_devel image +inputs: + github-token: + description: GitHub Token for registry access + required: true +outputs: + ip: + description: The IP of the tools_awx_1 container + value: ${{ steps.data.outputs.ip }} + admin-token: + description: OAuth token for admin user + value: ${{ steps.data.outputs.admin_token }} +runs: + using: composite + steps: + - name: Get python version from Makefile + shell: bash + run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV + + - name: Log in to registry + shell: bash + run: | + echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pre-pull latest available devel image and build HEAD on top of it + shell: bash + run: | + docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \ + COMPOSE_TAG=${{ github.base_ref }} \ + make docker-compose-build diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index e7c05ca9958f..7de3801080bd 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -5,9 +5,6 @@ name: Run AWX (devel environment) description: Runs AWX with `make docker-compose` inputs: - github-token: - description: GitHub Token for registry access - required: true build-ui: description: Should the UI be built? required: false @@ -23,10 +20,6 @@ outputs: runs: using: composite steps: - - name: Get python version from Makefile - shell: bash - run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV - - name: Upgrade ansible-core shell: bash run: python3 -m pip install --upgrade ansible-core @@ -35,19 +28,6 @@ runs: shell: bash run: sudo apt-get install -y gettext - - name: Log in to registry - shell: bash - run: | - echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - - name: Pre-pull latest available devel image and build HEAD on top of it - shell: bash - run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} - DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \ - COMPOSE_TAG=${{ github.base_ref }} \ - make docker-compose-build - - name: Start AWX shell: bash run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e69903b25288..ef559ebfd7be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,16 +37,32 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_ci_setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run check ${{ matrix.tests.name }} - run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make github_ci_runner + run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make docker-runner dev-env: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_ci_setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - uses: ./.github/actions/run_awx_devel + id: awx + with: + build-ui: false + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run smoke test - run: make github_ci_setup && ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v + run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v awx-operator: runs-on: ubuntu-latest @@ -131,6 +147,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_ci_setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/run_awx_devel id: awx with: diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 899afecd79f4..2a97728da7a3 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -21,6 +21,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_ci_setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/run_awx_devel id: awx with: diff --git a/Makefile b/Makefile index 6f08a6018b78..25005bdc9d6c 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ I18N_FLAG_FILE = .i18n_built sdist \ ui-release ui-devel \ VERSION PYTHON_VERSION docker-compose-sources \ - .git/hooks/pre-commit github_ci_setup github_ci_runner + .git/hooks/pre-commit clean-tmp: rm -rf tmp/ @@ -324,21 +324,10 @@ test: cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3 awx-manage check_migrations --dry-run --check -n 'missing_migration_file' -## Login to Github container image registry, pull image, then build image. -github_ci_setup: - # GITHUB_ACTOR is automatic github actions env var - # CI_GITHUB_TOKEN is defined in .github files - echo $(CI_GITHUB_TOKEN) | docker login ghcr.io -u $(GITHUB_ACTOR) --password-stdin - docker pull $(DEVEL_IMAGE_NAME) || : # Pre-pull image to warm build cache - $(MAKE) docker-compose-build - ## Runs AWX_DOCKER_CMD inside a new docker container. docker-runner: docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD) -## Builds image and runs AWX_DOCKER_CMD in it, mainly for .github checks. -github_ci_runner: github_ci_setup docker-runner - test_collection: rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt if [ "$(VENV_BASE)" ]; then \ diff --git a/tools/docker-compose/ansible/smoke-test.yml b/tools/docker-compose/ansible/smoke-test.yml index f4002c8567e2..591ceed39674 100644 --- a/tools/docker-compose/ansible/smoke-test.yml +++ b/tools/docker-compose/ansible/smoke-test.yml @@ -11,23 +11,6 @@ - name: Test that the development environment is able to launch a job hosts: localhost tasks: - - name: Boot the development environment - command: | - make docker-compose - environment: - COMPOSE_UP_OPTS: -d - args: - chdir: "{{ playbook_dir }}/../../../" - - # Takes a while for migrations to finish - - name: Wait for the dev environment to be ready - uri: - url: "http://localhost:8013/api/v2/ping/" - register: _result - until: _result.status == 200 - retries: 120 - delay: 5 - - name: Reset admin password shell: | docker exec -i tools_awx_1 bash < Date: Wed, 6 Sep 2023 15:00:56 -0400 Subject: [PATCH 2/5] Fix token parameter and remove outdated comments --- .github/actions/awx_ci_setup/action.yml | 11 +++++------ .github/actions/run_awx_devel/action.yml | 6 +----- .github/workflows/ci.yml | 2 -- .github/workflows/e2e_test.yml | 1 - 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/actions/awx_ci_setup/action.yml b/.github/actions/awx_ci_setup/action.yml index 552d3e61dda7..d097d76ef803 100644 --- a/.github/actions/awx_ci_setup/action.yml +++ b/.github/actions/awx_ci_setup/action.yml @@ -1,7 +1,3 @@ -# This currently *always* uses the "warm build cache" image -# We should do something to allow forcing a rebuild, probably by looking for -# some string in the commit message or something. - name: Setup images for AWX description: Builds new awx_devel image inputs: @@ -27,10 +23,13 @@ runs: run: | echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: Pre-pull latest available devel image and build HEAD on top of it + - name: Pre-pull latest devel image to warm cache + shell: bash + run: docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} + + - name: Build image for current source checkout shell: bash run: | - docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \ COMPOSE_TAG=${{ github.base_ref }} \ make docker-compose-build diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index 7de3801080bd..0172b900d0b1 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -1,8 +1,4 @@ -# This currently *always* uses the "warm build cache" image -# We should do something to allow forcing a rebuild, probably by looking for -# some string in the commit message or something. - -name: Run AWX (devel environment) +name: Run AWX docker-compose description: Runs AWX with `make docker-compose` inputs: build-ui: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef559ebfd7be..2b90daad2199 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,6 @@ jobs: id: awx with: build-ui: false - github-token: ${{ secrets.GITHUB_TOKEN }} - name: Run smoke test run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v @@ -156,7 +155,6 @@ jobs: id: awx with: build-ui: false - github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies for running tests run: | diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 2a97728da7a3..34b0a76fff00 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -30,7 +30,6 @@ jobs: id: awx with: build-ui: true - github-token: ${{ secrets.GITHUB_TOKEN }} log-filename: e2e-${{ matrix.job }}.log - name: Pull awx_cypress_base image From 2281725d8dd7048daaf1442f9593be0727e8613a Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 21 Sep 2023 13:46:11 -0400 Subject: [PATCH 3/5] Rename image action, nest under run_awx_devel --- .../{awx_ci_setup => awx_devel_image}/action.yml | 7 ------- .github/actions/run_awx_devel/action.yml | 8 ++++++++ .github/workflows/ci.yml | 14 +++----------- .github/workflows/e2e_test.yml | 7 +------ 4 files changed, 12 insertions(+), 24 deletions(-) rename .github/actions/{awx_ci_setup => awx_devel_image}/action.yml (80%) diff --git a/.github/actions/awx_ci_setup/action.yml b/.github/actions/awx_devel_image/action.yml similarity index 80% rename from .github/actions/awx_ci_setup/action.yml rename to .github/actions/awx_devel_image/action.yml index d097d76ef803..8b7081292bab 100644 --- a/.github/actions/awx_ci_setup/action.yml +++ b/.github/actions/awx_devel_image/action.yml @@ -4,13 +4,6 @@ inputs: github-token: description: GitHub Token for registry access required: true -outputs: - ip: - description: The IP of the tools_awx_1 container - value: ${{ steps.data.outputs.ip }} - admin-token: - description: OAuth token for admin user - value: ${{ steps.data.outputs.admin_token }} runs: using: composite steps: diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index 0172b900d0b1..955b0d58d586 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -6,6 +6,9 @@ inputs: required: false default: false type: boolean + github-token: + description: GitHub Token to pass to awx_devel_image + required: true outputs: ip: description: The IP of the tools_awx_1 container @@ -16,6 +19,11 @@ outputs: runs: using: composite steps: + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_devel_image + with: + github-token: ${{ inputs.github-token }} + - name: Upgrade ansible-core shell: bash run: python3 -m pip install --upgrade ansible-core diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b90daad2199..7612aa61c2aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v3 - name: Build awx_devel image for running checks - uses: ./.github/actions/awx_ci_setup + uses: ./.github/actions/awx_devel_image with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -50,15 +50,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Build awx_devel image for running checks - uses: ./.github/actions/awx_ci_setup - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/run_awx_devel id: awx with: build-ui: false + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Run smoke test run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v @@ -146,15 +142,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Build awx_devel image for running checks - uses: ./.github/actions/awx_ci_setup - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/run_awx_devel id: awx with: build-ui: false + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies for running tests run: | diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 34b0a76fff00..fa7ee3e429c9 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -21,16 +21,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Build awx_devel image for running checks - uses: ./.github/actions/awx_ci_setup - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/run_awx_devel id: awx with: build-ui: true - log-filename: e2e-${{ matrix.job }}.log + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Pull awx_cypress_base image run: | From 23f21026456212746d72c0f16120b2e14cd5fb7c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 21 Sep 2023 13:47:27 -0400 Subject: [PATCH 4/5] Move code back for git diff --- .github/actions/run_awx_devel/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index 955b0d58d586..9d4c85e60083 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -1,14 +1,14 @@ name: Run AWX docker-compose description: Runs AWX with `make docker-compose` inputs: + github-token: + description: GitHub Token to pass to awx_devel_image + required: true build-ui: description: Should the UI be built? required: false default: false type: boolean - github-token: - description: GitHub Token to pass to awx_devel_image - required: true outputs: ip: description: The IP of the tools_awx_1 container From 93a32d8d6f55214bce8a41c6832c9049f8dc77b8 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 21 Sep 2023 13:49:04 -0400 Subject: [PATCH 5/5] Fix indent --- .github/actions/run_awx_devel/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/run_awx_devel/action.yml b/.github/actions/run_awx_devel/action.yml index 9d4c85e60083..80b02669cf82 100644 --- a/.github/actions/run_awx_devel/action.yml +++ b/.github/actions/run_awx_devel/action.yml @@ -19,10 +19,10 @@ outputs: runs: using: composite steps: - - name: Build awx_devel image for running checks - uses: ./.github/actions/awx_devel_image - with: - github-token: ${{ inputs.github-token }} + - name: Build awx_devel image for running checks + uses: ./.github/actions/awx_devel_image + with: + github-token: ${{ inputs.github-token }} - name: Upgrade ansible-core shell: bash