diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml deleted file mode 100644 index bcbd05335..000000000 --- a/.github/workflows/rebase.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Automatic Rebase -on: - issue_comment: - types: [created] - pull_request_target: - types: [auto_merge_enabled] -env: - AUTO_REBASE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} -jobs: - check_token: - if: ${{ (github.event.issue.pull_request != '' && - contains(github.event.comment.body, '/rebase')) || github.event_name == - 'pull_request_target' }} - runs-on: ubuntu-latest - steps: - - name: Prompt user to add a token - uses: peter-evans/create-or-update-comment@v4 - with: - issue-number: ${{ github.event.issue.number || github.event.number }} - body: | - To automatically rebase, you need to add a [personal access token](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) with the name `AUTO_REBASE_PERSONAL_ACCESS_TOKEN` - to the [secrets section of this repo](https://github.com/${{ github.event.repository.full_name }}/settings/secrets/actions). - Once this is done, you can try the `/rebase` command again. - if: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN == '' }} - - name: Add -1 reaction - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ github.event.comment.id }} - reactions: "-1" - if: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN == '' && - github.event.comment }} - - name: Add +1 reaction - uses: peter-evans/create-or-update-comment@v4 - with: - comment-id: ${{ github.event.comment.id }} - reactions: "+1" - if: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN != '' && - github.event.comment }} - rebase: - if: ${{ github.event.issue.pull_request != '' && - contains(github.event.comment.body, '/rebase') || github.event_name == - 'pull_request_target' }} - name: Rebase - needs: check_token - runs-on: ubuntu-latest - steps: - - name: Checkout the latest code - uses: actions/checkout@v4 - with: - token: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - if: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN != '' }} - - name: Automatic Rebase - uses: cirrus-actions/rebase@1.8 - env: - GITHUB_TOKEN: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} - if: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN != '' }} - - name: Send failure message - uses: peter-evans/create-or-update-comment@v4 - with: - issue-number: ${{ github.event.issue.number || github.event.number }} - body: | - Automatic rebasing for this issue has failed :disappointed: - You'll have to rebase this one manually, I'm afraid. - if: ${{ failure() }} diff --git a/.github/workflows/trigger_rebase.yml b/.github/workflows/trigger_rebase.yml deleted file mode 100644 index 734a5c98a..000000000 --- a/.github/workflows/trigger_rebase.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Trigger Rebase - -on: - push: - branches: - - develop -env: - AUTO_REBASE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} -jobs: - fetch_oldest_pr_awaiting_automerge: - runs-on: ubuntu-latest - outputs: - pr_numbers: ${{ steps.get-all-awaiting-pr-numbers.outputs.pr_numbers }} - steps: - - name: Get all awaiting PR numbers - id: get-all-awaiting-pr-numbers - run: | - PR_NUMBER=$(curl https://api.github.com/repos/UKGovernmentBEIS/beis-report-official-development-assistance/pulls\?state\=open | jq -c '[.[] | select(.auto_merge!=null) | .number] | first') - echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT - trigger_rebase_action: - if: ${{ needs.fetch_oldest_pr_awaiting_automerge.outputs.pr_number != '' }} - env: - PR_NUMBER: ${{ needs.fetch_oldest_pr_awaiting_automerge.outputs.pr_number }} - needs: fetch_oldest_pr_awaiting_automerge - runs-on: ubuntu-latest - steps: - - name: Get the latest ref for ${{ env.PR_NUMBER }} - run: | - PR_DATA=$(curl https://api.github.com/repos/UKGovernmentBEIS/beis-report-official-development-assistance/pulls/$PR_NUMBER) - printf 'ref=%q\n' "$(echo "$PR_DATA" | jq -r .head.ref)" >> "$GITHUB_ENV" - - name: Checkout the latest code for ${{ env.PR_NUMBER }} - uses: actions/checkout@v4 - with: - token: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - ref: ${{ env.ref }} - - name: Automatic Rebase - uses: cirrus-actions/rebase@1.8 - env: - GITHUB_TOKEN: ${{ env.AUTO_REBASE_PERSONAL_ACCESS_TOKEN }} diff --git a/.node-version b/.node-version index 2a4e4ab81..7af24b7dd 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -16.17.0 +22.11.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b6410677..2bd50a94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ [Full changelog][unreleased] +## Release 153 - 2024-12-02 + +[Full changelog][153] + +- Update Node to 22.11.0 + ## Release 152 - 2024-11-27 [Full changelog][152] @@ -1713,7 +1719,8 @@ - Planned start and end dates are mandatory - Actual start and end dates must not be in the future -[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-152...HEAD +[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-153...HEAD +[153]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-152...release-153 [152]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-151...release-152 [151]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-150...release-151 [150]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-149...release-150 diff --git a/Dockerfile b/Dockerfile index 3c376c47c..4d4e11d50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ ARG NODE_MAJOR ENV APP_HOME=/app ENV DEPS_HOME=/deps -ENV NODE_MAJOR=${NODE_MAJOR:-16} +ENV NODE_MAJOR=${NODE_MAJOR:-22} ENV RAILS_ENV=${RAILS_ENV:-production} ENV NODE_ENV=${RAILS_ENV:-production} diff --git a/Gemfile b/Gemfile index 5f78336da..bb0e50ffe 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ ruby File.read(".ruby-version").strip gem "acts_as_tree" gem "addressable" gem "audited", "~> 5.4" -gem "aws-sdk-s3", "~> 1.173" +gem "aws-sdk-s3", "~> 1.174" gem "bootsnap", ">= 1.1.0", require: false gem "cssbundling-rails", "~> 1.4" gem "csv-safe" @@ -25,7 +25,7 @@ gem "nanoid" gem "notifications-ruby-client" gem "parser" gem "pry-rails" -gem "puma", "~> 6.4" +gem "puma", "~> 6.5" gem "pundit" gem "rollbar" gem "rails", "~> 6" diff --git a/Gemfile.lock b/Gemfile.lock index ef9813f11..76eef1919 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -71,8 +71,8 @@ GEM activerecord (>= 5.0, < 7.2) request_store (~> 1.2) aws-eventstream (1.3.0) - aws-partitions (1.1012.0) - aws-sdk-core (3.213.0) + aws-partitions (1.1013.0) + aws-sdk-core (3.214.0) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.992.0) aws-sigv4 (~> 1.9) @@ -80,7 +80,7 @@ GEM aws-sdk-kms (1.96.0) aws-sdk-core (~> 3, >= 3.210.0) aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.173.0) + aws-sdk-s3 (1.174.0) aws-sdk-core (~> 3, >= 3.210.0) aws-sdk-kms (~> 1) aws-sigv4 (~> 1.5) @@ -223,7 +223,8 @@ GEM jsbundling-rails (1.3.1) railties (>= 6.0.0) json (2.6.3) - jwt (2.7.1) + jwt (2.9.3) + base64 language_server-protocol (3.17.0.3) launchy (3.0.1) addressable (~> 2.8) @@ -250,12 +251,12 @@ GEM net-imap net-pop net-smtp - mail-notify (1.2.0) - actionmailer (>= 5.2.4.6) - actionpack (>= 5.2.7.1) - actionview (>= 5.2.7.1) - activesupport (>= 5.2.4.6) - notifications-ruby-client (~> 5.1) + mail-notify (2.0.0) + actionmailer (>= 5.2.8.1) + actionpack (>= 5.2.8.1) + actionview (>= 5.2.8.1) + activesupport (>= 5.2.8.1) + notifications-ruby-client (~> 6.0) rack (>= 2.1.4.1) marcel (1.0.4) matrix (0.4.2) @@ -293,7 +294,7 @@ GEM racc (~> 1.4) nokogiri (1.16.7-x86_64-linux) racc (~> 1.4) - notifications-ruby-client (5.4.0) + notifications-ruby-client (6.2.0) jwt (>= 1.5, < 3) observer (0.1.2) orm_adapter (0.5.0) @@ -311,7 +312,7 @@ GEM pry-rails (0.3.11) pry (>= 0.13.0) public_suffix (5.1.1) - puma (6.4.3) + puma (6.5.0) nio4r (~> 2.0) pundit (2.4.0) activesupport (>= 3.0.0) @@ -443,7 +444,7 @@ GEM ruby_parser (3.19.1) sexp_processor (~> 4.16) rubyzip (2.3.2) - selenium-webdriver (4.26.0) + selenium-webdriver (4.27.0) base64 (~> 0.2) logger (~> 1.4) rexml (~> 3.2, >= 3.2.5) @@ -534,7 +535,7 @@ GEM railties (>= 3.0.7) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.18) + zeitwerk (2.7.1) PLATFORMS arm64-darwin-21 @@ -546,7 +547,7 @@ DEPENDENCIES acts_as_tree addressable audited (~> 5.4) - aws-sdk-s3 (~> 1.173) + aws-sdk-s3 (~> 1.174) better_errors binding_of_caller bootsnap (>= 1.1.0) @@ -587,7 +588,7 @@ DEPENDENCIES pg pry-byebug pry-rails - puma (~> 6.4) + puma (~> 6.5) pundit pundit-matchers (~> 3.1.2) rack-attack diff --git a/README.md b/README.md index 1fd91a501..f2b665a1a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/workflows/CI/badge.svg?branch=develop)](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/actions?query=branch%3Adevelop) +[![CI Checks](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/actions/workflows/ci_checks.yml/badge.svg?branch=develop)](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/actions/workflows/ci_checks.yml) [![Coverage Status](https://coveralls.io/repos/github/UKGovernmentBEIS/beis-report-official-development-assistance/badge.svg?branch=develop)](https://coveralls.io/github/UKGovernmentBEIS/beis-report-official-development-assistance?branch=develop) # Report Official Development Assistance (RODA) @@ -12,45 +12,28 @@ Technology (DSIT), which had a large impact on the application. You will see references to both organisations in both the code and the data, and this is worth keeping in mind as you work on the application. -## Environments -The app is currently hosted on AWS ECS clusters provided by DSIT. The -deployments happen automatically through AWS CodePipeline pipelines, after a -successful GitHub Action build on the target GitHub branch. +## Environments and hosting +All environments are hosting on AWS provided by DSIT, for more details see the +[hosting documentation.](/doc/hosting.md) -**NOTE**: The AWS pipelines are polling-based. If a pipeline has not run in over -30 days, AWS will turn it off, and a developer will have to log in and restart -deployments manually, by using "Release change". +To access the environments you will need a `digital-paas-prodction-account` +account and then [assume the correct role](/doc/hosting.md#assuming-roles) for +each environment. -### Dev +### Environments -[https://dev.report-official-development-assistance.service.gov.uk](https://dev.report-official-development-assistance.service.gov.uk) - -The `develop` branch is deployed to the dev environment. - -### Staging - -[https://staging.report-official-development-assistance.service.gov.uk](https://staging.report-official-development-assistance.service.gov.uk) - -The `develop` branch is deployed to the staging environment. - -### Training - -[http://training.report-official-development-assistance.service.gov.uk](https://training.report-official-development-assistance.service.gov.uk) - -The `main` branch is deployed to the training environment. - -### Production - -[https://www.report-official-development-assistance.service.gov.uk](https://www.report-official-development-assistance.service.gov.uk) - -The `main` branch is deployed to the production environment. +- [Development](https://dev.report-official-development-assistance.service.gov.uk) +- [Staging](https://staging.report-official-development-assistance.service.gov.uk) +- [Training](https://training.report-official-development-assistance.service.gov.uk) +- [Production](https://www.report-official-development-assistance.service.gov.uk) ## Getting started If this is your first time running the application, see the [getting started documentation](/doc/getting-started.md) for instructions. -## Day to day development +You will likely need access to the environments, see the [hosting +documentation](/doc/hosting.md) on how to obtain this. ## Support documentation @@ -67,7 +50,11 @@ documentation](/doc/getting-started.md) for instructions. ## In-depth documentation - [Glossary of business terms](/doc/glossary.md) +- [Hosting](/doc/hosting.md) +- [Deployment](/doc/deployment-process.md) +- [Logging](/doc/logging.md) - [Environment variables](/doc/environment-variables.md) +- [Feature flags](/doc/feature-flags.md) - [Sending email notifications](/doc/email-notifications.md) - [Use authentication](/doc/authentication.md) - [Importing new partner organisation @@ -83,8 +70,6 @@ documentation](/doc/getting-started.md) for instructions. - [Identifying invalid activities](/doc/utilities.md) - [Background jobs](/doc/background-jobs.md) - [Console access](/doc/console-access.md) -- [Logging](/doc/logging.md) -- [Deployment](/doc/deployment-process.md) - [DNS](/doc/dns.md) - [Migrations](/doc/migrations.md) diff --git a/app/views/shared/xml/_activity.xml.haml b/app/views/shared/xml/_activity.xml.haml index f30d1c746..b198acae9 100644 --- a/app/views/shared/xml/_activity.xml.haml +++ b/app/views/shared/xml/_activity.xml.haml @@ -23,6 +23,10 @@ %narrative= organisation.name - if activity.previous_identifier? %other-identifier{"ref" => activity.previous_identifier, type: "A1"} + - if activity.hybrid_beis_dsit_activity? + %other-identifier{"ref" => "GB-GOV-13", type: "B1"} + %owner-org{"ref" => "GB-GOV-13"} + %narrative{"xml:lang" => activity.organisation.language_code}= "DSIT previous reporting-org identifier" %activity-status{"code" => activity.iati_status}/ - if activity.planned_start_date? %activity-date{"iso-date" => activity.planned_start_date, type: "1"}/ diff --git a/db/data/20241114082139_fix_hybrid_beis_dsit_identifier.rb b/db/data/20241114082139_fix_hybrid_beis_dsit_identifier.rb new file mode 100644 index 000000000..0e0c18cbf --- /dev/null +++ b/db/data/20241114082139_fix_hybrid_beis_dsit_identifier.rb @@ -0,0 +1,65 @@ +# Require me in the console with `require Rails.root + "db/data/20241114082139_fix_hybrid_beis_dsit_identifier.rb"` +# then run me with `FixHybridBeisDsitIdentifier.new.migrate!` +# +# Description: +# +# For every Activity where `previous_identifier` is NOT nil +# When the `previous_identifier` starts "GB-GOV-13" AND `transparency_identifier` starts "GB-GOV-26" +# Then set `transparency_identifier` to start "GB-GOV-13" +# AND set `previous_identifier` to nil +# AND set hybrid_beis_dsit_activity to true. +# +class FixHybridBeisDsitIdentifier + attr_reader :target, :updated + + def initialize + @target = 0 + @updated = 0 + end + + def migrate! + target_activities.each do |activity| + fix_activity(activity) + end + + puts "Total target activities: #{@target}" + puts "Total updated activities: #{@updated}" + + true + end + + def target_activities + activities = Activity.where.not(previous_identifier: nil).select do |activity| + identifier_starts_with_beis(activity.previous_identifier) && + identifier_starts_with_dsit(activity.transparency_identifier) + end + + @target = activities.count + activities + end + + def fix_activity(activity) + puts "Updating activity id #{activity.id} with IATI ID: #{activity.transparency_identifier} and Previous IATI ID: #{activity.previous_identifier}" + + activity.update!( + transparency_identifier: activity.previous_identifier, + previous_identifier: nil, + hybrid_beis_dsit_activity: true + ) + + @updated += 1 + puts "Activity Updated!" + end + + def identifier_starts_with_beis(identifier) + return if identifier.nil? + + identifier.slice(0, 9).eql?("GB-GOV-13") + end + + def identifier_starts_with_dsit(identifier) + return if identifier.nil? + + identifier.slice(0, 9).eql?("GB-GOV-26") + end +end diff --git a/db/migrate/20241114114012_add_hybrid_beis_dsit_activity.rb b/db/migrate/20241114114012_add_hybrid_beis_dsit_activity.rb new file mode 100644 index 000000000..73801767d --- /dev/null +++ b/db/migrate/20241114114012_add_hybrid_beis_dsit_activity.rb @@ -0,0 +1,5 @@ +class AddHybridBeisDsitActivity < ActiveRecord::Migration[6.1] + def change + add_column :activities, :hybrid_beis_dsit_activity, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index 7d6542e2b..3355aa58a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_01_15_183402) do +ActiveRecord::Schema.define(version: 2024_11_14_114012) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" @@ -84,6 +84,7 @@ t.string "ispf_oda_partner_countries", array: true t.string "ispf_non_oda_partner_countries", array: true t.string "spending_breakdown_filename" + t.boolean "hybrid_beis_dsit_activity", default: false t.index ["extending_organisation_id"], name: "index_activities_on_extending_organisation_id" t.index ["level"], name: "index_activities_on_level" t.index ["organisation_id"], name: "index_activities_on_organisation_id" diff --git a/doc/console-access.md b/doc/console-access.md index 3ddb37cac..7c775bf6f 100644 --- a/doc/console-access.md +++ b/doc/console-access.md @@ -1,3 +1,71 @@ # Console access -Currently, dxw developers don't have access to the live application console. +Console access to the running containers is via a 'Jumphost' in AWS, you will need +a `digital-paas-production-account`, see [hosting +documentation](/doc/hosting.md#getting-an-account) for instructions for obtaining this. + +Once you have access, you can access a console by following these steps: + +- sign in to AWS with your `digital-paas-production-account` account +- assume the role for the appropriate environment, see [hosting + documentation](/doc/hosting.md#assuming-roles) + +You will need to make a note of the cluster name, ARN of the task and the +container name: + +First the cluster name: + +- Locate _Elastic Container Service_ (ECS) in the services menu +- You will see only a single cluster with a name that matches the environment, + e.g. `ODA-dev` for development +- Note down the cluster name + +Next, the task ARN: + +- Click on the cluster +- Switch to the 'Tasks' tab +- Click on the copy icon for the first Task and make a note of it, it will be a + long ARN string + +And finally, the container name: + +- Click on the task you copied the ARN of in the previous step +- Look at the list of containers, you will see the application and a 'SideCar', + make a note of the application container name, this is usually `ODA-app` + +You now have everything you need to use the console: + +- Locate _EC2_ in the services menu +- In the sidebar, click on 'Instances' +- You will see a single 'Jumphost' instance for the environment, e.g. + `Jumphost-dev` for development +- Check that row and click the _Connect_ button +- Switch to the _Session Manager_ tab +- Click on the _Connect_ button +- You will see a console on the Jumphost + +You now have to connect to the container itself from the Jumphost: + +Use the following command: + +``` +aws ecs execute-command --region [region] --cluster [cluster name] --task [task +arn] --container [container name] --command [command] --interactive +``` + +Where: + +* `[region]`: `eu-west-2` +* `[cluster name]`: The cluster name you have noted down +* `[task arn]`: The task ARN you have noted down +* `[container name]`: The container name you have noted down +* `[command]`: `/bin/bash` + +AWS documentation for the [aws ecs +execute-command](https://docs.aws.amazon.com/cli/latest/reference/ecs/execute-command.html) + +## Troubleshooting + +- If you cannot see anything in ECS or EC2, you likely need to [assume the correct + role](/doc/hosting.md#assuming-roles) +- All variables are case sensitive diff --git a/doc/deployment-process.md b/doc/deployment-process.md index ac0592320..710592837 100644 --- a/doc/deployment-process.md +++ b/doc/deployment-process.md @@ -1,86 +1,148 @@ -# Deployment process +# Deployment -## Production +## Introduction -Production deploys are done by manually merging develop into main. To give us a slightly more formal process around what gets deployed and when and also to give us visibility into the things that have been deployed, we additionally follow these steps when releasing to production: +Deployments are made from AWS via CodePipelines. -Releases are documented in the [CHANGELOG](../CHANGELOG.md) following the [Keep a changelog](https://keepachangelog.com/en/1.0.0/) format. +To view deployments or trigger them, you will need access to the hosting, see +the [hosting documentation](/doc/hosting.md) for more information. -When a new release is deployed to production, a new second-level heading should be created in CHANGELOG.md with the release number and details of what has changed in this release. +**IMPORTANT**: The CodePipelines are polling-based. If a pipeline has not run in over +30 days, AWS will turn it off, and a developer will have to [trigger a +release](#monitor-or-trigger-deployments) manually. -The heading should link to a Github URL at the bottom of the file, which shows the differences between the current release and the previous one. For example: +**NOTE**: There is no indication in Github what branch is deployed where or any +deployment status, the only way to find out is via the CodePipelines in AWS. -### Example +## Environments + +Whilst we have four environments, they are linked together in pairs, which +limits their value: + +Merges into the `develop` branch will deploy the changes to: + +- development +- staging + +Merging into the `main` branch will deploy the changes to: + +- training +- production + + +## Monitor or trigger deployments + +### Monitoring + +To monitor deployments follow these steps: + +- sign in to AWS with your `digital-paas-production-account` account +- assume the role for the appropriate environment, see [hosting + documentation](/doc/hosting.md#assuming-roles) +- Locate _CodePipelines_ in the services menu +- Click on the name of the Pipeline + +You will see the three stages of deployment: Source, Build and Deploy. You can +view details of each stage. + +### Triggering + +To trigger a deployment: + +- sign in to AWS with your `digital-paas-production-account` account +- assume the role for the appropriate environment, see [hosting + documentation](/doc/hosting.md#assuming-roles) +- Locate _CodePipelines_ in the services menu +- Click on the name of the Pipeline +- Click on the _Release changes_ button, top right + +The branch will be deployed, see [environments](#environments) + +## Release process + +Production deploys are done by manually merging develop into main. To give us a +slightly more formal process around what gets deployed and when and also to give +us visibility into the things that have been deployed, we additionally follow +these steps when releasing to production: + +Releases are documented in the [CHANGELOG](/CHANGELOG.md) following the [Keep +a changelog](https://keepachangelog.com/en/1.0.0/) format. + +When a new release is deployed to production, a new second-level heading should +be created in CHANGELOG.md with the release number and details of what has +changed in this release. + +The heading should link to a Github URL at the bottom of the file, which shows +the differences between the current release and the previous one. For example: ``` -## [release-1] +## Release-1 - 2024-01-01 + +[Full changelog][1] + - A change - Another change -[release-1]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-1...release-0 +[unreleased]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-1...HEAD +[1]: https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/compare/release-0...release-1 ``` -### Steps +### Steps 1. Confirm the release candidate and perform any prerequisites - - Confirm the release with any relevant people (product owner, delivery manager, etc) - - Think about any dependencies that also need considering: dependent parts of the service that also need updating; environment variables that need changing/adding; third-party services that need to be set up/updated; data migrations to be run + - Confirm the release with any relevant people (product owner, delivery + manager, etc) + - Think about any dependencies that also need considering: dependent parts of + the service that also need updating; environment variables that need + changing/adding; third-party services that need to be set up/updated; data + migrations to be run 1. Create a release branch and make a pull request - - Create a branch from develop for the release called release-X where X is the release number + - Create a branch from develop for the release called release-X where X is + the release number - Update CHANGELOG.md to: - document the changes in this release in a bullet point form - add a link to the diff at the bottom of the file - Copy the list of changes from the changelog into the commit message - - Push the changes up to Github - ``` - git push -u origin release-X - # e.g. git push -u origin release-120 - ```` - - Create a pull request to merge that release into **develop** with content from the CHANGELOG.md + - Push the changes up to Github + ``` + git push -u origin release-X # e.g. git + push -u origin release-120 + ``` + - Create a pull request to merge that release into _develop_ with content + from the CHANGELOG.md 1. Get that pull request reviewed and merged - - Confirm that the changes in the release are safe to ship and that CHANGELOG.md accurately reflects the changes included in the release. + - Confirm that the changes in the release are safe to ship and that + CHANGELOG.md accurately reflects the changes included in the release. - Merge the pull request -1. Update your local develop branch and tag the merge commit on develop - ``` - git tag release-X [merge-commit-for-release] - # e.g. `git tag release-120 e1156bfd2cf45a0281808edb8342055407c0f253` - ``` -1. Push the tag to Github (we need the refs otherwise git will not know - if you mean the tag or the branch as they have the same name) - ``` - git push origin refs/tags/release-X - # e.g. `git push origin refs/tags/release-120` - ``` -1. Announce the release - Let the team know about the release. This is posted in Slack under #beis-roda. Typical form is: - ``` - @here :badgerbadger: Release N of RODA going to production :badgerbadger: - ``` +1. Update your local develop branch and tag the merge commit on develop + ``` + git tag release-X [merge-commit-for-release] + ``` +1. Push the tag to Github (we need the refs otherwise git will not know if you +mean the tag or the branch as they have the same name) + ``` + git push --tags + ``` +1. Create a Github release + - Click on Releases + - Click on Draft new release + - Choose the release tag + - Set the release title to 'Release-x' + - Set the body of the release to list of changes for the release, copy them + from the CHANGELOG.md + - Click on Publish release 1. Manually merge develop into main in order to release - - Once the release pull request has been merged into the develop branch, the production deploy can be performed by manually merging develop into main: - ``` - git fetch - git checkout main - git pull - git merge origin/develop - # Edit the commit message to reference the release number - # e.g. "Release 43" or "merge origin/develop for release 43" - git push - ``` -1. Production smoke test - Once the code has been deployed to production, carry out a quick smoke test to confirm that the changes have been successfully deployed. -1. [Run any data migrations](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance#data--one-off-tasks) that are meant to be run as part of the release -1. Move all the Trello cards from "Awaiting release" to "Done" - -## Staging - -1. Open a pull request back into the `develop` branches with your changes -1. Get that pull request code reviewed and approved -1. Check that any prerequisite changes to things like environment variables or third-party service configuration is ready -1. Merge the pull request - -The changes should be automatically applied by Github Actions. [You can track the progress of Github Actions jobs at this link](https://github.com/UKGovernmentBEIS/beis-report-official-development-assistance/actions?query=workflow%3ADeploy). - -## Migrations - -These should be automatically run when new containers are started during a deployment. This instruction is included in the app/docker-entrypoint.sh. + - Once the release pull request has been merged into the develop branch, the + production deploy can be performed by manually merging develop into main: + ``` + git fetch + git checkout main + git pull + git merge origin/develop + ``` + - Set the merge commit title to 'Release-x' +1. Production smoke test once the code has been deployed to production, carry +out a quick smoke test to confirm that the changes have been successfully +deployed. +1. Announce the release Let the team know about the release. This is posted in +Slack under #beis-roda, link to the release notes. diff --git a/doc/feature-flags.md b/doc/feature-flags.md new file mode 100644 index 000000000..748753be6 --- /dev/null +++ b/doc/feature-flags.md @@ -0,0 +1,7 @@ +# Feature flags + +The application uses the [Rollout gem](https://github.com/FetLife/rollout) to +provide and manage feature flags. + +Rollout provides a UI at `/rollout` to service owner users. + diff --git a/doc/hosting.md b/doc/hosting.md new file mode 100644 index 000000000..99df118a5 --- /dev/null +++ b/doc/hosting.md @@ -0,0 +1,61 @@ +# Hosting + +## Introduction + +All the environments are hosted on AWS provided by DSIT ICS. + +This means we have no direct control of how the infrastructure is setup or +managed, depending on what your goal is, you may have to work with DSIT ICS. + +## Getting an account + +You will first need to get a `digital-paas-production-account`. Which will need +to be done via DIST. dxw support may be able to help, but work with your +delivery manager and DSIT colleges first. + +Once you get an account you will have: + +- username +- password +- MFA +- signin url + +Once signed in you will need to assume a role for each environment, see below. + +## Services + +A brief overview of the AWS services used: + +- Elastic Container Service (ECS) +- EC2 +- Elastic Container Registry (ECR) +- CodeBuild +- CodeDeploy +- CodePipeline +- CloudWatch + +For details on how deployments work, see the [deployment +documentation.](/doc/deployment-process.md) + +For details of how to get a console in an environment, see the [console access +documentation.](/doc/console-access.md) + +## Assuming roles + +Each environment has its own role that you will need to assume in order to +interact with it. + +The role variables are stored in the `RODA` vault in the dxw 1Password, you need +access to it first. + +Once you have a `digital-paas-production-account` account and 1Password vault +access, sign into AWS and assume the role for the appropriate environment: + +- From the user menu, top right of the window, click on the _Switch role_ button +- Locate the RODA 1Password vault and the `AWS environment roles` note +- copy and paste the values from the note to the Switch Role form +- click on the 'Switch role' button + +You will see that your _role_ has changed in the top right of the window, you +can now view the environment, check deploys, connect to the console, etc. + diff --git a/doc/logging.md b/doc/logging.md index b83680bdf..7f0dbb6e7 100644 --- a/doc/logging.md +++ b/doc/logging.md @@ -1,3 +1,38 @@ # Logging -Logs can be accessed in the AWS ECS ODA clusters, under the ODA services corresponding to the target environment. +Logging is handled by CloudWatch in the DSIT AWS, you will need +a `digital-paas-production-account`, see [hosting +documentation](/doc/hosting.md#getting-an-account) for instructions for obtaining this. + +Once you have access you can find the logs by following these steps: + +- sign in to AWS with your `digital-paas-production-account` account +- assume the role for the appropriate environment, see [hosting + documentation](/doc/hosting.md#assuming-roles) +- Locate _Elastic Container Service_ (ECS) in the services menu +- Click on the cluster +- Click on the service +- Switch to the _Logs_ tab +- Refine the view of the logs + +You can then switch to CloudWatch if desired: + +- Click the _View in CloudWatch_ drop down +- Select the container +- Refine the view of the logs + +## Via CloudWatch + +You can access the container logs directly via CloudWatch, with some additional +steps: + +- sign in to AWS with your `digital-paas-production-account` account +- assume the role for the appropriate environment, see [hosting + documentation](/doc/hosting.md#assuming-roles) +- Locate _CloudWatch_ in the services menu +- From the side bar open _Logs_ and click on _Log groups_ +- Locate either the application or worker container for that matches the + environment, e.g. `/ecs/ODA-prod` and `/ecs/ODA-prod-SideCar` for production +- Click on the container +- The logs are split into 'streams' choose a steam or click _Search all log + steams_ and refine diff --git a/spec/db/data/20241114082139_fix_hybrid_beis_dsit_identifier_spec.rb b/spec/db/data/20241114082139_fix_hybrid_beis_dsit_identifier_spec.rb new file mode 100644 index 000000000..152725ec7 --- /dev/null +++ b/spec/db/data/20241114082139_fix_hybrid_beis_dsit_identifier_spec.rb @@ -0,0 +1,98 @@ +require Rails.root + "db/data/20241114082139_fix_hybrid_beis_dsit_identifier.rb" + +RSpec.describe FixHybridBeisDsitIdentifier do + describe "#migrate!" do + it "updates only the appropriate activities and includes a count" do + activity_to_update = create( + :programme_activity, + transparency_identifier: "GB-GOV-26-AAAA-BBBB-CCC-DDDDDDD", + previous_identifier: "GB-GOV-13-AAAA-BBBB-CCC-DDDDDDD", + hybrid_beis_dsit_activity: false + ) + activity_to_ignore = create( + :programme_activity, + transparency_identifier: "GB-GOV-26-EEEE-FFFF-GGG-HHHHHHH", + previous_identifier: nil, + hybrid_beis_dsit_activity: false + ) + other_activity_to_ignore = create( + :programme_activity, + transparency_identifier: "GB-GOV-13-EEEE-FFFF-GGG-HHHHHHH", + previous_identifier: "GB-GOV-13-IIII_JJJJ_KKK_LL", + hybrid_beis_dsit_activity: false + ) + + migration = described_class.new + migration.migrate! + + expect(activity_to_update.reload.transparency_identifier).to eql "GB-GOV-13-AAAA-BBBB-CCC-DDDDDDD" + expect(activity_to_update.hybrid_beis_dsit_activity).to be true + + expect(activity_to_ignore.reload.transparency_identifier).to eql "GB-GOV-26-EEEE-FFFF-GGG-HHHHHHH" + expect(activity_to_ignore.hybrid_beis_dsit_activity).to be false + + expect(other_activity_to_ignore.reload.transparency_identifier).to eql "GB-GOV-13-EEEE-FFFF-GGG-HHHHHHH" + expect(other_activity_to_ignore.hybrid_beis_dsit_activity).to be false + + expect(migration.target).to be 1 + expect(migration.updated).to be 1 + end + end + + describe "#identifier_starts_with_beis" do + it "returns true for identifiers that start GB-GOV-13" do + valid_identifier = "GB-GOV-13-1234-5678-91011" + + expect(described_class.new.identifier_starts_with_beis(valid_identifier)).to be true + end + + it "returns false for identifiers that do not start GB-GOV-13" do + valid_identifier = "GB-GOV-10-1234-5678-91011" + + expect(described_class.new.identifier_starts_with_beis(valid_identifier)).to be false + end + end + + describe "#identifier_starts_with_dsit" do + it "returns true for identifiers that start GB-GOV-26" do + valid_identifier = "GB-GOV-26-1234-5678-91011" + + expect(described_class.new.identifier_starts_with_dsit(valid_identifier)).to be true + end + + it "returns false for identifiers that do not start GB-GOV-26" do + valid_identifier = "GB-GOV-10-1234-5678-91011" + + expect(described_class.new.identifier_starts_with_dsit(valid_identifier)).to be false + end + end + + describe "#fix_activity" do + let(:activity) do + create( + :programme_activity, + transparency_identifier: "GB-GOV-26-AAAA-BBBB-CCC-DDDDDDD", + previous_identifier: "GB-GOV-13-AAAA-BBBB-CCC-DDDDDDD", + hybrid_beis_dsit_activity: false + ) + end + + it "updates the transparency_identifier to the value of previous_identifier" do + described_class.new.fix_activity(activity) + + expect(activity.transparency_identifier).to eql "GB-GOV-13-AAAA-BBBB-CCC-DDDDDDD" + end + + it "updates the previous_identifier to nil" do + described_class.new.fix_activity(activity) + + expect(activity.previous_identifier).to be_nil + end + + it "updates the hybrid BEIS DSIT state to true" do + described_class.new.fix_activity(activity) + + expect(activity.hybrid_beis_dsit_activity).to be true + end + end +end diff --git a/spec/views/shared/xml/activity_spec.rb b/spec/views/shared/xml/activity_spec.rb index c6a82d761..0c386f64a 100644 --- a/spec/views/shared/xml/activity_spec.rb +++ b/spec/views/shared/xml/activity_spec.rb @@ -30,6 +30,28 @@ end end + describe "Other identifier for BEIS to DSIT" do + context "when hybrid_beis_dsit_activity is true" do + let(:activity) { create(:programme_activity, hybrid_beis_dsit_activity: true) } + let!(:commitment) { nil } + + it "includes the other-identifier element" do + expect(rendered).to include("") + expect(rendered).to include("") + expect(rendered).to include("DSIT previous reporting-org identifier") + end + end + end + describe "activity-scope" do context "when there are benefitting countries" do let(:commitment) { nil }