Add latest changes from gitlab-org/gitlab@master

app/controllers/projects/pipelines/application_controller.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Abstract class encapsulating common logic for creating new controllers in a pipeline context
+
+module Projects
+  module Pipelines
+    class ApplicationController < Projects::ApplicationController
+      include Gitlab::Utils::StrongMemoize
+
+      before_action :pipeline
+      before_action :authorize_read_pipeline!
+
+      private
+
+      def pipeline
+        strong_memoize(:pipeline) do
+          project.all_pipelines.find(params[:pipeline_id]).tap do |pipeline|
+            render_404 unless can?(current_user, :read_pipeline, pipeline)
+          end
+        end
+      end
+    end
+  end
+end

app/graphql/types/project_type.rb

@@ -251,7 +251,8 @@ class ProjectType < BaseObject
           null: true,
           description: 'A single release of the project',
           resolver: Resolvers::ReleasesResolver.single,
-          feature_flag: :graphql_release_data
+          feature_flag: :graphql_release_data,
+          authorize: :download_code
 
     field :container_expiration_policy,
           Types::ContainerExpirationPolicyType,

app/graphql/types/release_assets_type.rb

@@ -3,14 +3,15 @@
 module Types
   class ReleaseAssetsType < BaseObject
     graphql_name 'ReleaseAssets'
+    description 'A container for all assets associated with a release'
 
     authorize :read_release
 
     alias_method :release, :object
 
     present_using ReleasePresenter
 
-    field :assets_count, GraphQL::INT_TYPE, null: true,
+    field :count, GraphQL::INT_TYPE, null: true, method: :assets_count,
           description: 'Number of assets of the release'
     field :links, Types::ReleaseLinkType.connection_type, null: true,
           description: 'Asset links of the release'

app/graphql/types/release_link_type.rb

@@ -3,6 +3,7 @@
 module Types
   class ReleaseLinkType < BaseObject
     graphql_name 'ReleaseLink'
+    description 'Represents an asset link associated with a release'
 
     authorize :read_release
 

app/graphql/types/release_source_type.rb

@@ -3,8 +3,9 @@
 module Types
   class ReleaseSourceType < BaseObject
     graphql_name 'ReleaseSource'
+    description 'Represents the source code attached to a release in a particular format'
 
-    authorize :read_release_sources
+    authorize :download_code
 
     field :format, GraphQL::STRING_TYPE, null: true,
           description: 'Format of the source'

app/graphql/types/release_type.rb

@@ -3,17 +3,20 @@
 module Types
   class ReleaseType < BaseObject
     graphql_name 'Release'
+    description 'Represents a release'
 
     authorize :read_release
 
     alias_method :release, :object
 
     present_using ReleasePresenter
 
-    field :tag_name, GraphQL::STRING_TYPE, null: false, method: :tag,
-          description: 'Name of the tag associated with the release'
+    field :tag_name, GraphQL::STRING_TYPE, null: true, method: :tag,
+          description: 'Name of the tag associated with the release',
+          authorize: :download_code
     field :tag_path, GraphQL::STRING_TYPE, null: true,
-          description: 'Relative web path to the tag associated with the release'
+          description: 'Relative web path to the tag associated with the release',
+          authorize: :download_code
     field :description, GraphQL::STRING_TYPE, null: true,
           description: 'Description (also known as "release notes") of the release'
     markdown_field :description_html, null: true
@@ -39,8 +42,7 @@ def author
 
     field :commit, Types::CommitType, null: true,
           complexity: 10, calls_gitaly: true,
-          description: 'The commit associated with the release',
-          authorize: :reporter_access
+          description: 'The commit associated with the release'
 
     def commit
       return if release.sha.nil?

app/models/ci/build.rb

@@ -27,7 +27,7 @@ class Build < Ci::Processable
       upload_multiple_artifacts: -> (build) { build.publishes_artifacts_reports? },
       refspecs: -> (build) { build.merge_request_ref? },
       artifacts_exclude: -> (build) { build.supports_artifacts_exclude? },
-      release_steps: -> (build) { build.release_steps? }
+      multi_build_steps: -> (build) { build.multi_build_steps? }
     }.freeze
 
     DEFAULT_RETRIES = {
@@ -890,7 +890,7 @@ def supports_artifacts_exclude?
         Gitlab::Ci::Features.artifacts_exclude_enabled?
     end
 
-    def release_steps?
+    def multi_build_steps?
       options.dig(:release)&.any? &&
         Gitlab::Ci::Features.release_generation_enabled?
     end

app/policies/releases/source_policy.rb

@@ -3,11 +3,5 @@
 module Releases
   class SourcePolicy < BasePolicy
     delegate { @subject.project }
-
-    rule { can?(:public_access) | can?(:reporter_access) }.policy do
-      enable :read_release_sources
-    end
-
-    rule { ~can?(:read_release) }.prevent :read_release_sources
   end
 end

app/presenters/release_presenter.rb

@@ -5,7 +5,7 @@ class ReleasePresenter < Gitlab::View::Presenter::Delegated
 
   presents :release
 
-  delegate :project, :tag, :assets_count, to: :release
+  delegate :project, :tag, to: :release
 
   def commit_path
     return unless release.commit && can_download_code?
@@ -43,6 +43,18 @@ def edit_url
     edit_project_release_url(project, release)
   end
 
+  def assets_count
+    if can_download_code?
+      release.assets_count
+    else
+      release.assets_count(except: [:sources])
+    end
+  end
+
+  def name
+    can_download_code? ? release.name : "Release-#{release.id}"
+  end
+
   private
 
   def can_download_code?

changelogs/unreleased/215160-add-intermediate-group-deploy-key-table.yml

@@ -0,0 +1,5 @@
+---
+title: Create group_deploy_keys_groups intermediate table
+merge_request: 32901
+author:
+type: added

db/migrate/20200522205606_create_group_deploy_keys_group.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateGroupDeployKeysGroup < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+  DOWNTIME = false
+
+  def up
+    with_lock_retries do
+      create_table :group_deploy_keys_groups do |t|
+        t.timestamps_with_timezone
+
+        t.references :group, index: false, null: false, foreign_key: { to_table: :namespaces, on_delete: :cascade }
+        t.references :group_deploy_key, null: false, foreign_key: { on_delete: :cascade }
+
+        t.index [:group_id, :group_deploy_key_id], unique: true, name: 'index_group_deploy_keys_group_on_group_deploy_key_and_group_ids'
+      end
+    end
+  end
+
+  def down
+    with_lock_retries do
+      # rubocop:disable Migration/DropTable
+      drop_table :group_deploy_keys_groups
+      # rubocop:enable Migration/DropTable
+    end
+  end
+end

db/structure.sql

@@ -3208,6 +3208,23 @@ CREATE TABLE public.group_deploy_keys (
     CONSTRAINT check_f58fa0a0f7 CHECK ((char_length(key) <= 4096))
 );
 
+CREATE TABLE public.group_deploy_keys_groups (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    group_id bigint NOT NULL,
+    group_deploy_key_id bigint NOT NULL
+);
+
+CREATE SEQUENCE public.group_deploy_keys_groups_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.group_deploy_keys_groups_id_seq OWNED BY public.group_deploy_keys_groups.id;
+
 CREATE SEQUENCE public.group_deploy_keys_id_seq
     START WITH 1
     INCREMENT BY 1
@@ -7759,6 +7776,8 @@ ALTER TABLE ONLY public.group_custom_attributes ALTER COLUMN id SET DEFAULT next
 
 ALTER TABLE ONLY public.group_deploy_keys ALTER COLUMN id SET DEFAULT nextval('public.group_deploy_keys_id_seq'::regclass);
 
+ALTER TABLE ONLY public.group_deploy_keys_groups ALTER COLUMN id SET DEFAULT nextval('public.group_deploy_keys_groups_id_seq'::regclass);
+
 ALTER TABLE ONLY public.group_deploy_tokens ALTER COLUMN id SET DEFAULT nextval('public.group_deploy_tokens_id_seq'::regclass);
 
 ALTER TABLE ONLY public.group_group_links ALTER COLUMN id SET DEFAULT nextval('public.group_group_links_id_seq'::regclass);
@@ -8569,6 +8588,9 @@ ALTER TABLE ONLY public.group_custom_attributes
 ALTER TABLE ONLY public.group_deletion_schedules
     ADD CONSTRAINT group_deletion_schedules_pkey PRIMARY KEY (group_id);
 
+ALTER TABLE ONLY public.group_deploy_keys_groups
+    ADD CONSTRAINT group_deploy_keys_groups_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY public.group_deploy_keys
     ADD CONSTRAINT group_deploy_keys_pkey PRIMARY KEY (id);
 
@@ -10032,6 +10054,10 @@ CREATE INDEX index_group_deletion_schedules_on_marked_for_deletion_on ON public.
 
 CREATE INDEX index_group_deletion_schedules_on_user_id ON public.group_deletion_schedules USING btree (user_id);
 
+CREATE UNIQUE INDEX index_group_deploy_keys_group_on_group_deploy_key_and_group_ids ON public.group_deploy_keys_groups USING btree (group_id, group_deploy_key_id);
+
+CREATE INDEX index_group_deploy_keys_groups_on_group_deploy_key_id ON public.group_deploy_keys_groups USING btree (group_deploy_key_id);
+
 CREATE UNIQUE INDEX index_group_deploy_keys_on_fingerprint ON public.group_deploy_keys USING btree (fingerprint);
 
 CREATE INDEX index_group_deploy_keys_on_fingerprint_sha256 ON public.group_deploy_keys USING btree (fingerprint_sha256);
@@ -12746,6 +12772,9 @@ ALTER TABLE ONLY public.project_repositories
 ALTER TABLE ONLY public.packages_nuget_dependency_link_metadata
     ADD CONSTRAINT fk_rails_c3313ee2e4 FOREIGN KEY (dependency_link_id) REFERENCES public.packages_dependency_links(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY public.group_deploy_keys_groups
+    ADD CONSTRAINT fk_rails_c3854f19f5 FOREIGN KEY (group_deploy_key_id) REFERENCES public.group_deploy_keys(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.merge_request_user_mentions
     ADD CONSTRAINT fk_rails_c440b9ea31 FOREIGN KEY (note_id) REFERENCES public.notes(id) ON DELETE CASCADE;
 
@@ -12884,6 +12913,9 @@ ALTER TABLE ONLY public.merge_request_metrics
 ALTER TABLE ONLY public.draft_notes
     ADD CONSTRAINT fk_rails_e753681674 FOREIGN KEY (merge_request_id) REFERENCES public.merge_requests(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY public.group_deploy_keys_groups
+    ADD CONSTRAINT fk_rails_e87145115d FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY public.description_versions
     ADD CONSTRAINT fk_rails_e8f4caf9c7 FOREIGN KEY (epic_id) REFERENCES public.epics(id) ON DELETE CASCADE;
 
@@ -13987,6 +14019,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200521225327
 20200521225337
 20200521225346
+20200522205606
 20200522235146
 20200525114553
 20200525121014

doc/administration/gitaly/praefect.md

@@ -739,9 +739,17 @@ current primary node is found to be unhealthy.
 It is likely that we will implement support for Consul, and a cloud native
 strategy in the future.
 
-## Identifying Impact of a Primary Node Failure
+## Primary Node Failure
 
-When a primary Gitaly node fails, there is a chance of data loss. Data loss can occur if there were outstanding replication jobs the secondaries did not manage to process before the failure. The `dataloss` Praefect sub-command helps identify these cases by counting the number of dead replication jobs for each repository. This command must be executed on a Praefect node.
+Praefect recovers from a failing primary Gitaly node by promoting a healthy secondary as the new primary. To minimize data loss, Praefect elects the secondary with the least unreplicated writes from the primary. There can still be some unreplicated writes, leading to data loss.
+
+Praefect switches a virtual storage in to read-only mode after a failover event. This eases data recovery efforts by preventing new, possibly conflicting writes to the newly elected primary. This allows the administrator to attempt recovering the lost data before allowing new writes.
+
+If you prefer write availability over consistency, this behavior can be turned off by setting `praefect['failover_read_only_after_failover'] = false` in `/etc/gitlab/gitlab.rb` and [reconfiguring Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure).
+
+### Checking for data loss
+
+The Praefect `dataloss` sub-command helps identify lost writes by counting the number of dead replication jobs for each repository within a given time frame. This command must be executed on a Praefect node.
 
 A time frame to search can be specified with `-from` and `-to`:
 
@@ -769,6 +777,29 @@ sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.t
 To check a project's repository checksums across on all Gitaly nodes, run the
 [replicas Rake task](../raketasks/praefect.md#replica-checksums) on the main GitLab node.
 
+### Recovering lost writes
+
+The Praefect `reconcile` sub-command can be used to recover lost writes from the
+previous primary once it is back online. This is only possible when the virtual storage
+is still in read-only mode.
+
+```shell
+sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml reconcile -virtual <virtual-storage> -reference <previous-primary> -target <current-primary> -f
+```
+
+Refer to [Backend Node Recovery](#backend-node-recovery) section for more details on
+the `reconcile` sub-command.
+
+### Enabling Writes
+
+Any data recovery attempts should have been made before enabling writes to eliminate
+any chance of conflicting writes. Virtual storage can be re-enabled for writes by using
+the Praefect `enable-writes` sub-command.
+
+```shell
+sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml enable-writes -virtual-storage <virtual-storage>
+```
+
 ## Backend Node Recovery
 
 When a Praefect backend node fails and is no longer able to

doc/api/graphql/reference/gitlab_schema.graphql

@@ -10102,6 +10102,9 @@ enum RegistryState {
   SYNCED
 }
 
+"""
+Represents a release
+"""
 type Release {
   """
   Assets of the release
@@ -10196,19 +10199,22 @@ type Release {
   """
   Name of the tag associated with the release
   """
-  tagName: String!
+  tagName: String
 
   """
   Relative web path to the tag associated with the release
   """
   tagPath: String
 }
 
+"""
+A container for all assets associated with a release
+"""
 type ReleaseAssets {
   """
   Number of assets of the release
   """
-  assetsCount: Int
+  count: Int
 
   """
   Asset links of the release
@@ -10356,6 +10362,9 @@ type ReleaseEvidenceEdge {
   node: ReleaseEvidence
 }
 
+"""
+Represents an asset link associated with a release
+"""
 type ReleaseLink {
   """
   Indicates the link points to an external resource
@@ -10443,6 +10452,9 @@ enum ReleaseLinkType {
   RUNBOOK
 }
 
+"""
+Represents the source code attached to a release in a particular format
+"""
 type ReleaseSource {
   """
   Format of the source