From c457430dbb521ef36fc844058d2c5c3a2289a642 Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 12 Nov 2024 12:09:32 +0000 Subject: [PATCH 1/6] Basic linting on Dockerfile --- Dockerfile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index e1fbd7e2..633969a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # ------------------------------------------------------------------------------ # Base # ------------------------------------------------------------------------------ -FROM ruby:3.3.6 as base +FROM ruby:3.3.6 AS base LABEL org.opencontainers.image.authors="contact@dxw.com" COPY .node-version .node-version @@ -17,12 +17,12 @@ RUN \ build-essential \ libpq-dev -ENV APP_HOME /srv/app -ENV DEPS_HOME /deps +ENV APP_HOME=/srv/app +ENV DEPS_HOME=/deps ARG RAILS_ENV -ENV RAILS_ENV ${RAILS_ENV:-production} -ENV NODE_ENV ${RAILS_ENV:-production} +ENV RAILS_ENV=${RAILS_ENV:-production} +ENV NODE_ENV=${RAILS_ENV:-production} # ------------------------------------------------------------------------------ # Dependencies @@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y yarn WORKDIR ${DEPS_HOME} # Install Ruby dependencies -ENV BUNDLE_GEM_GROUPS ${RAILS_ENV} +ENV BUNDLE_GEM_GROUPS=${RAILS_ENV} COPY Gemfile ${DEPS_HOME}/Gemfile COPY Gemfile.lock ${DEPS_HOME}/Gemfile.lock @@ -109,8 +109,8 @@ RUN \ ARG CURRENT_GIT_SHA ARG TIME_OF_BUILD -ENV CURRENT_GIT_SHA ${CURRENT_GIT_SHA} -ENV TIME_OF_BUILD ${TIME_OF_BUILD} +ENV CURRENT_GIT_SHA=${CURRENT_GIT_SHA} +ENV TIME_OF_BUILD=${TIME_OF_BUILD} COPY ./docker-entrypoint.sh / RUN chmod +x /docker-entrypoint.sh @@ -123,7 +123,7 @@ CMD ["bundle", "exec", "rails", "server"] # ------------------------------------------------------------------------------ # Test # ------------------------------------------------------------------------------ -FROM web as test +FROM web AS test RUN \ apt-get update && \ From 5f83437b622126c1e2552a19d512fc19a1ee00b8 Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 12 Nov 2024 09:38:25 +0000 Subject: [PATCH 2/6] Initial update to CI We wanted to try and speed up the CI tasks for the Rails Template. This work introduces the same approach we took to save time on the DfE Complete project. We use Docker supplied actions to build the image and cache the layers to the Github actions cache. Subsequent jobs are set to use the layers from the cache which, in most cases, gives us a nice speed boost. We've split the jobs so they can be run in parallel, as we know the image is the same for each job they all use the strategy. We've kept the Shellcheck task separate as this code is not strictly speaking the application code, so doesn't really need to be inside the image. --- .github/workflows/continuous-integration.yml | 129 +++++++++++++++++-- 1 file changed, 121 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 3e5a0a40..358d8e8f 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -1,5 +1,7 @@ # TODO: Enable GitHub Actions on the repository to test all pull requests # https://github.com///actions +name: CI Checks + on: pull_request: push: @@ -8,18 +10,129 @@ on: - develop jobs: - test: + build-and-cache: + name: Build and cache image runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and cache image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: RAILS_ENV=test + push: false + tags: app_test:latest + cache-from: type=gha + cache-to: type=gha,mode=min - env: - RAILS_ENV: test + lint-and-format: + name: Lint and format application + runs-on: ubuntu-latest + needs: build-and-cache + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and cache + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + RAILS_ENV=test + push: false + load: true + tags: app_test:latest + cache-from: type=gha + - + name: Run linters and formatters + run: | + docker run --rm app_test:latest /bin/bash -c "bundle exec standardrb -f simple && \ + yarn run lint:js && \ + yarn run lint:css && \ + yarn run lint:format" + static-analysis: + name: Static analysis + runs-on: ubuntu-latest + needs: build-and-cache steps: - - name: Check out code + - + name: Checkout uses: actions/checkout@v4 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and cache + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + RAILS_ENV=test + push: false + load: true + tags: app_test:latest + cache-from: type=gha + - + name: Run Brakeman + run: | + docker run --rm app_test:latest /bin/bash -c "bundle exec brakeman -o /dev/stdout" - - name: Build - run: script/ci/cibuild + specs: + name: Specs and coverage + runs-on: ubuntu-latest + needs: build-and-cache + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and cache + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + RAILS_ENV=test + push: false + load: true + tags: app_test:latest + cache-from: type=gha + - + name: Run RSpec and Simplecov + run: | + docker compose -p complete-app -f docker-compose.ci.yml \ + run --name app_test test /bin/bash -c "bin/rails spec" + - + name: Shutdown containers + run: docker compose -p app_test down && docker compose -p app_test rm - - name: Test - run: script/ci/test + shellcheck: + name: Lint scripts + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Run Shellceck + run: | + for file in $(git ls-files script/*) + do shellcheck -x "$file" + done From 706294dc80a026c3a8d82c11394271683b110178 Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 12 Nov 2024 14:46:32 +0000 Subject: [PATCH 3/6] Remove unused CI scripts Now that we have all the CI tasks in Github Actions these scripts are never used. --- script/ci/cibuild | 8 -------- script/ci/test | 12 ------------ 2 files changed, 20 deletions(-) delete mode 100755 script/ci/cibuild delete mode 100755 script/ci/test diff --git a/script/ci/cibuild b/script/ci/cibuild deleted file mode 100755 index 36639123..00000000 --- a/script/ci/cibuild +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# Setup environment for CI to run tests. This is primarily designed to run on -# the continuous integration server. - -set -e - -docker compose -f docker-compose.ci.yml -p app build diff --git a/script/ci/test b/script/ci/test deleted file mode 100755 index db27458c..00000000 --- a/script/ci/test +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# Run the test suite for the application. This is primarily designed to run on -# CI with a clean build context. To run locally first remove any generic image -# by running `docker rmi app:latest` - -set -e - -docker compose -f docker-compose.ci.yml \ - -p app \ - run --rm test \ - script/all/test "$@" From 56915a20d15b486c5e229e0c6861e247fe201e29 Mon Sep 17 00:00:00 2001 From: Ynda Jas Date: Thu, 14 Nov 2024 12:13:44 +0000 Subject: [PATCH 4/6] DRY up cache loading in CI workflow This adds a composite action to run the two cache-loading steps that are reused across three of the jobs in the continuous integration workflow. The second of these steps is a little long and detailed, and differs slightly but meaningfully from a similar step in the build cache job, so it might be useful to DRY this up. This also allows us to see the meat of the post-cache jobs a little easier in the continuous integration workflow The `actions/checkout@v4` step is needed in each job in order to load our action (and presumably also the external ones used in the composite action) It would be quite nice to use a YAML anchor or alias to do this kind of reuse, but these are currently unsupported in GitHub Actions. They might be on the way soon, so watch this space: https://github.com/actions/runner/issues/1182 --- .../load-cache/action.yml | 19 ++++++++ .github/workflows/continuous-integration.yml | 48 +++---------------- 2 files changed, 25 insertions(+), 42 deletions(-) create mode 100644 .github/actions/continuous-integration/load-cache/action.yml diff --git a/.github/actions/continuous-integration/load-cache/action.yml b/.github/actions/continuous-integration/load-cache/action.yml new file mode 100644 index 00000000..f1b1085d --- /dev/null +++ b/.github/actions/continuous-integration/load-cache/action.yml @@ -0,0 +1,19 @@ +name: Build and cache image +runs: + using: "composite" + steps: + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - + name: Build and cache image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + build-args: | + RAILS_ENV=test + push: false + load: true + tags: app_test:latest + cache-from: type=gha diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 358d8e8f..8ecc7fae 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -41,20 +41,8 @@ jobs: name: Checkout uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and cache - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - build-args: | - RAILS_ENV=test - push: false - load: true - tags: app_test:latest - cache-from: type=gha + name: Load cache + uses: ./.github/actions/continuous-integration/load-cache - name: Run linters and formatters run: | @@ -72,20 +60,8 @@ jobs: name: Checkout uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and cache - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - build-args: | - RAILS_ENV=test - push: false - load: true - tags: app_test:latest - cache-from: type=gha + name: Load cache + uses: ./.github/actions/continuous-integration/load-cache - name: Run Brakeman run: | @@ -100,20 +76,8 @@ jobs: name: Checkout uses: actions/checkout@v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and cache - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - build-args: | - RAILS_ENV=test - push: false - load: true - tags: app_test:latest - cache-from: type=gha + name: Load cache + uses: ./.github/actions/continuous-integration/load-cache - name: Run RSpec and Simplecov run: | From 3ad3c736fb646b72bc796e1207d9b6a8b2becfe4 Mon Sep 17 00:00:00 2001 From: meyric Date: Mon, 18 Nov 2024 09:01:57 +0000 Subject: [PATCH 5/6] Remove independent build and cache job Our caching strategy is to cached across CI jobs with the focus on caching the layers in the Docker image. As each job has to build the image from (using the cache) regardless, we think we can get rid of the independent build and cache job and let the caching happen as we run each job. The only way to see if this pays off is to merge the changes and run another CI workflow to see how much caching we get. --- .../load-cache/action.yml | 1 + .github/workflows/continuous-integration.yml | 31 ++----------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/actions/continuous-integration/load-cache/action.yml b/.github/actions/continuous-integration/load-cache/action.yml index f1b1085d..d72ba42c 100644 --- a/.github/actions/continuous-integration/load-cache/action.yml +++ b/.github/actions/continuous-integration/load-cache/action.yml @@ -17,3 +17,4 @@ runs: load: true tags: app_test:latest cache-from: type=gha + cache-to: type=gha,mode=min diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 8ecc7fae..039074ee 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -10,38 +10,15 @@ on: - develop jobs: - build-and-cache: - name: Build and cache image - runs-on: ubuntu-latest - steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and cache image - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - build-args: RAILS_ENV=test - push: false - tags: app_test:latest - cache-from: type=gha - cache-to: type=gha,mode=min - lint-and-format: name: Lint and format application runs-on: ubuntu-latest - needs: build-and-cache steps: - name: Checkout uses: actions/checkout@v4 - - name: Load cache + name: Build and cache uses: ./.github/actions/continuous-integration/load-cache - name: Run linters and formatters @@ -54,13 +31,12 @@ jobs: static-analysis: name: Static analysis runs-on: ubuntu-latest - needs: build-and-cache steps: - name: Checkout uses: actions/checkout@v4 - - name: Load cache + name: Build and cache uses: ./.github/actions/continuous-integration/load-cache - name: Run Brakeman @@ -70,13 +46,12 @@ jobs: specs: name: Specs and coverage runs-on: ubuntu-latest - needs: build-and-cache steps: - name: Checkout uses: actions/checkout@v4 - - name: Load cache + name: Build and cache uses: ./.github/actions/continuous-integration/load-cache - name: Run RSpec and Simplecov From b33d141376aa65f08db7607c9de3d952f699f69d Mon Sep 17 00:00:00 2001 From: meyric Date: Fri, 13 Dec 2024 12:49:06 +0000 Subject: [PATCH 6/6] Split linting and formatting runs Split each linter and formatter into their own `run` steps, this is for clarity. --- .github/workflows/continuous-integration.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 039074ee..1cac2da2 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -21,12 +21,21 @@ jobs: name: Build and cache uses: ./.github/actions/continuous-integration/load-cache - - name: Run linters and formatters + name: Run Ruby standard run: | - docker run --rm app_test:latest /bin/bash -c "bundle exec standardrb -f simple && \ - yarn run lint:js && \ - yarn run lint:css && \ - yarn run lint:format" + docker run --rm app_test:latest /bin/bash -c "bundle exec standardrb -f simple" + - + name: Run ESLint + run: | + docker run --rm app_test:latest /bin/bash -c "yarn run lint:js" + - + name: Run Stylelint + run: | + docker run --rm app_test:latest /bin/bash -c "yarn run lint:css" + - + name: Run Prettier + run: | + docker run --rm app_test:latest /bin/bash -c "yarn run lint:format" static-analysis: name: Static analysis