From 969926b9cf4491f262a659b5ce63827e81949aec Mon Sep 17 00:00:00 2001 From: meyric Date: Thu, 21 Nov 2024 15:23:04 +0000 Subject: [PATCH 1/8] Stop compiling assets outside production If we use `bin/rails spec` to run our specs in CI we will not need to compile the assets - this should save some time building the image for CI. The reason is that the `spec` task includes `test:prepare` which builds the assets in test on the fly. --- .github/workflows/ci_checks.yml | 2 +- .rspec | 1 + Dockerfile | 10 +++------- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_checks.yml b/.github/workflows/ci_checks.yml index e1a917ab5..c5102545c 100644 --- a/.github/workflows/ci_checks.yml +++ b/.github/workflows/ci_checks.yml @@ -54,7 +54,7 @@ jobs: name: Run Rspec and Simplecov run: | docker compose -p app_test -f docker-compose.ci.yml \ - run --name app_test test /bin/bash -c "bin/rspec --format=documentation" + run --name app_test test /bin/bash -c "bin/rails spec" - name: Copy coverage report from container run: mkdir coverage && docker cp app_test:/app/coverage/lcov.info coverage/lcov.info diff --git a/.rspec b/.rspec index a4a94f7c9..c821136e2 100644 --- a/.rspec +++ b/.rspec @@ -1,2 +1,3 @@ --require spec_helper --require rails_helper +--format documentation diff --git a/Dockerfile b/Dockerfile index 8a5a62e03..704774fa0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,13 +115,9 @@ COPY . ${APP_HOME} RUN cp -R $DEPS_HOME/node_modules $APP_HOME/node_modules RUN cp -R $DEPS_HOME/node_modules/govuk-frontend/govuk/assets $APP_HOME/app/assets -RUN \ - RAILS_ENV=$RAILS_ENV \ - DOMAIN="stand-in.local" \ - SECRET_KEY_BASE="super secret" \ - DATABASE_URL="postgres://stand-in:5432" \ - REDIS_URL="redis://stand-in.local:6379" \ - bundle exec rake assets:precompile --quiet +RUN if [ ${RAILS_ENV} = "production" ]; then \ + bundle exec rake assets:precompile --quiet; \ + fi # create tmp/pids RUN mkdir -p tmp/pids From 2ab2e62fc383d76bcad39e43c1bb2b020ed1fb6c Mon Sep 17 00:00:00 2001 From: meyric Date: Thu, 21 Nov 2024 15:46:34 +0000 Subject: [PATCH 2/8] Remove old CI We've moved over to our new CI Github action and this one no longer works as it relies on pre-compiled assets so we remove it along with the script it uses. --- .github/workflows/ci.yml | 23 ----------------------- script/cibuild | 9 --------- 2 files changed, 32 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100755 script/cibuild diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index f975c23a2..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: CI - -on: [push] - -env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Create coverage directory - run: mkdir -p ./coverage - - name: Run tests - run: script/cibuild - - name: Coveralls - uses: coverallsapp/github-action@v2 - with: - file: ./coverage/lcov.info - fail-on-error: false diff --git a/script/cibuild b/script/cibuild deleted file mode 100755 index 31987c1d7..000000000 --- a/script/cibuild +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -set -e - -docker compose -f docker-compose.ci.yml build -docker compose -f docker-compose.ci.yml run \ - -e CI=true \ - -e COVERALLS_REPO_TOKEN="$COVERALLS_REPO_TOKEN" \ - test script/test From 406f1dfda1295ea295c3665bda9099ef95cb8b3b Mon Sep 17 00:00:00 2001 From: meyric Date: Wed, 27 Nov 2024 09:23:00 +0000 Subject: [PATCH 3/8] Fix: Asset compilation We recently switched off asset compilation in environments other than production. In production, we thought the environment variables required for asset compilation would be present, when we deployed the change to the development environment, it turned out that was not the case (we have no direct control over how the infrastructure is configured). Returning these stand in variables allows the assets to be compiled, the variables are not used in the running container and this is how we approached this issue before the change. --- Dockerfile | 6 ++++++ README.md | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 704774fa0..3c376c47c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,7 +115,13 @@ COPY . ${APP_HOME} RUN cp -R $DEPS_HOME/node_modules $APP_HOME/node_modules RUN cp -R $DEPS_HOME/node_modules/govuk-frontend/govuk/assets $APP_HOME/app/assets +# We need a secret key, database url and Redis url to compile assets, these are +# not used in the running application RUN if [ ${RAILS_ENV} = "production" ]; then \ + DOMAIN="stand-in.local" \ + SECRET_KEY_BASE="super secret" \ + DATABASE_URL="postgres://stand-in:5432" \ + REDIS_URL="redis://stand-in.local:6379" \ bundle exec rake assets:precompile --quiet; \ fi diff --git a/README.md b/README.md index 1e31a7dae..1fd91a501 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,10 @@ documentation](/doc/getting-started.md) for instructions. ## Errors and monitoring -TBC +### Errors + +We send errors to a dxw owned [Rollbar +project](https://rollbar.com/dxw/dsit-roda/), contact dxw support to get access. ## Architecture decision records We use ADRs to document architectural decisions We use ADRs to document architectural decisions that we make. They can be found From a9838a1769cf4e1fe2e6ab43ee39c71e029a3888 Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 19 Nov 2024 11:52:19 +0000 Subject: [PATCH 4/8] Update Rollout config The call to `Redis.current` is deprecated and so we replace it with the default connection to Redis. --- config/initializers/rollout.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/config/initializers/rollout.rb b/config/initializers/rollout.rb index 25cca501e..57c26f0c1 100644 --- a/config/initializers/rollout.rb +++ b/config/initializers/rollout.rb @@ -1,6 +1,4 @@ -require "redis" - -ROLLOUT = Rollout.new(Redis.current) +ROLLOUT = Rollout.new(Redis.new(url: ENV["REDIS_URL"])) ROLLOUT.define_group(:beis_users) do |user| user.service_owner? From 6d7fd0bbffa07f3aa738b725d4d5ec6b143e288e Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 19 Nov 2024 12:02:28 +0000 Subject: [PATCH 5/8] Update Redis and Sidekiq Now that we've cleared the deprecated method call, these two gems can be updated. They have to go together because Sidekiq locks to Redis. --- CHANGELOG.md | 2 ++ Gemfile | 4 ++-- Gemfile.lock | 29 +++++++++++++++++------------ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64728ef74..895fa7d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ [Full changelog][unreleased] +- Update Redis and Sidekiq + ## Release 151 - 2024-11-26 [Full changelog][151] diff --git a/Gemfile b/Gemfile index 5d6e4399f..5f78336da 100644 --- a/Gemfile +++ b/Gemfile @@ -32,11 +32,11 @@ gem "rails", "~> 6" gem "rack-attack" gem "rollout" gem "rollout-ui" -gem "redis", "< 5" +gem "redis", "~> 5" gem "redis-namespace" gem "redis-actionpack" gem "redis-store" -gem "sidekiq", "~> 6.5.10" +gem "sidekiq", "~> 7" gem "tzinfo-data", platforms: %i[mingw mswin x64_mingw jruby] gem "wicked" gem "strip_attributes" diff --git a/Gemfile.lock b/Gemfile.lock index b57a6c626..ef9813f11 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -169,12 +169,11 @@ GEM railties (>= 5.0.0) faker (3.5.1) i18n (>= 1.8.11, < 2) - fakeredis (0.8.0) - redis (~> 4.1) + fakeredis (0.1.4) ffi (1.17.0) ffi (1.17.0-arm64-darwin) ffi (1.17.0-x86_64-darwin) - ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.0-x86_64-linux) foreman (0.88.1) globalid (1.2.1) activesupport (>= 6.1) @@ -296,6 +295,7 @@ GEM racc (~> 1.4) notifications-ruby-client (5.4.0) jwt (>= 1.5, < 3) + observer (0.1.2) orm_adapter (0.5.0) parallel (1.23.0) parser (3.3.6.0) @@ -371,11 +371,14 @@ GEM rb-fsevent (0.11.2) rb-inotify (0.11.1) ffi (~> 1.0) - redis (4.8.1) + redis (5.3.0) + redis-client (>= 0.22.0) redis-actionpack (5.5.0) actionpack (>= 5) redis-rack (>= 2.1.0, < 4) redis-store (>= 1.1.0, < 2) + redis-client (0.22.2) + connection_pool redis-namespace (1.11.0) redis (>= 4) redis-rack (3.0.0) @@ -393,8 +396,9 @@ GEM railties (>= 5.2) rexml (3.3.9) rollbar (3.6.0) - rollout (2.5.0) - redis (~> 4.0) + rollout (2.6.1) + observer + redis (>= 4.0, < 6) rollout-ui (0.5.3) rollout (~> 2.5) sinatra (~> 2.0) @@ -448,10 +452,11 @@ GEM sexp_processor (4.16.1) shoulda-matchers (6.1.0) activesupport (>= 5.2.0) - sidekiq (6.5.12) - connection_pool (>= 2.2.5, < 3) - rack (~> 2.0) - redis (>= 4.5.0, < 5) + sidekiq (7.3.6) + connection_pool (>= 2.3.0) + logger + rack (>= 2.2.4) + redis-client (>= 0.22.2) simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) @@ -589,7 +594,7 @@ DEPENDENCIES rails (~> 6) rails-controller-testing rails_layout - redis (< 5) + redis (~> 5) redis-actionpack redis-namespace redis-store @@ -599,7 +604,7 @@ DEPENDENCIES rspec-rails selenium-webdriver shoulda-matchers - sidekiq (~> 6.5.10) + sidekiq (~> 7) simplecov (~> 0.22.0) simplecov-lcov (~> 0.8.0) spring From a7e7c8c6411e6db23483bb93205d85a8a59f6d11 Mon Sep 17 00:00:00 2001 From: meyric Date: Fri, 22 Nov 2024 12:11:13 +0000 Subject: [PATCH 6/8] Update compose Redis We want to run Redis in CI rather than rely on the fake redis gem. We also want the major version of Redis to match that in production (7) locally and in CI. --- backing-services-docker-compose.yml | 2 +- docker-compose.ci.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backing-services-docker-compose.yml b/backing-services-docker-compose.yml index 0fec4eae1..4e2004688 100644 --- a/backing-services-docker-compose.yml +++ b/backing-services-docker-compose.yml @@ -9,7 +9,7 @@ services: volumes: - db_data:/var/lib/postgresql/data redis: - image: redis:4 + image: redis:7 ports: - "6379:6379" volumes: diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9e42e38bd..9dc8d3c5c 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -8,12 +8,16 @@ services: - app_test:latest depends_on: - db + - redis environment: DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL: "true" DATABASE_URL: postgres://postgres:password@db:5432/roda_test + REDIS_URL: redis://redis:6379 env_file: - .env.test db: image: postgres:13 environment: POSTGRES_PASSWORD: password + redis: + image: redis:7 From 6d02f70ed2f11c2684504e6ac7156ebfaab57ad9 Mon Sep 17 00:00:00 2001 From: meyric Date: Tue, 19 Nov 2024 12:03:34 +0000 Subject: [PATCH 7/8] Stop using Fake Redis gem We have Redis locally and in CI so we don't need to fake it. --- spec/rails_helper.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 44999e88f..b83032d09 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -8,7 +8,6 @@ abort("The Rails environment is running in production mode!") if Rails.env.production? require "rspec/rails" # Add additional requires below this line. Rails is not loaded until this point! -require "fakeredis/rspec" # testing strip_attributes require "strip_attributes/matchers" From 35bd1f490e9ea7aa884056de2ddb04cf7334396a Mon Sep 17 00:00:00 2001 From: meyric Date: Wed, 27 Nov 2024 12:24:07 +0000 Subject: [PATCH 8/8] Release 152 - Update Redis and Sidekiq --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 895fa7d19..7b6410677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ [Full changelog][unreleased] +## Release 152 - 2024-11-27 + +[Full changelog][152] + - Update Redis and Sidekiq ## Release 151 - 2024-11-26 @@ -1709,7 +1713,8 @@ - Planned start and end dates are mandatory - Actual start and end dates must not be in the future -[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-151...HEAD +[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-152...HEAD +[152]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-151...release-152 [151]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-150...release-151 [150]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-149...release-150 [149]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-148...release-149