feat(publish test results): only publish verification results when al…

…l interactions have been run

lib/pact/provider/rspec.rb

@@ -61,24 +61,24 @@ def describe_interaction_with_provider_state interaction, options
 
         def describe_interaction interaction, options
 
-          # pact_uri, pact_provider_state and pact_description are used by
+          # pact_uri and pact_interaction are used by
           # Pact::Provider::RSpec::PactBrokerFormatter
+
+          # pact_interaction_example_description is used by
+          # Pact::Provider::RSpec::Formatter and Pact::Provider::RSpec::Formatter2
+
+          # pact: verify is used to allow RSpec before and after hooks.
           metadata = {
             pact: :verify,
             pact_interaction: interaction,
             pact_interaction_example_description: interaction_description_for_rerun_command(interaction),
-            pact_json: options[:pact_json],
-            pact_uri: options[:pact_uri],
-            pact_provider_state: interaction.provider_state,
-            pact_description: interaction.description
+            pact_uri: options[:pact_uri]
           }
 
           describe description_for(interaction), metadata do
 
-
             describe "with #{interaction.request.method_and_path}" do
 
-
               interaction_context = InteractionContext.new
 
               before do | example |

lib/pact/provider/rspec/pact_broker_formatter.rb

@@ -53,11 +53,12 @@ def close(_notification)
         def format_example(example)
           {
             exampleDescription: example.description,
-            fullDescription: example.full_description,
+            exampleFullDescription: example.full_description,
             status: example.execution_result.status.to_s,
-            interactionProviderState: example.metadata[:pact_provider_state],
-            interactionDescription: example.metadata[:pact_description],
-            pact_uri: example.metadata[:pact_uri]
+            interactionProviderState: example.metadata[:pact_interaction].provider_state,
+            interactionDescription: example.metadata[:pact_interaction].description,
+            pact_uri: example.metadata[:pact_uri],
+            pact_interaction: example.metadata[:pact_interaction]
           }
         end
       end

lib/pact/provider/verification_results/create.rb

@@ -14,7 +14,7 @@ def initialize pact_source, test_results_hash
         end
 
         def call
-          VerificationResult.new(!any_failures?, Pact.configuration.provider.application_version, test_results_hash_for_pact_uri)
+          VerificationResult.new(publishable?, !any_failures?, Pact.configuration.provider.application_version, test_results_hash_for_pact_uri)
         end
 
         private
@@ -27,19 +27,32 @@ def any_failures?
           count_failures_for_pact_uri > 0
         end
 
+        def publishable?
+          executed_interactions_count == all_interactions_count && all_interactions_count > 0
+        end
+
         def examples_for_pact_uri
-          @examples_for_pact_uri ||= test_results_hash[:examples]
-                                      .select{ |e| e[:pact_uri] == pact_uri }
-                                      .collect{ |e| clean_example(e) }
+          @examples_for_pact_uri ||= test_results_hash[:examples].select{ |e| e[:pact_uri] == pact_uri }
         end
 
         def count_failures_for_pact_uri
           examples_for_pact_uri.count{ |e| e[:status] != 'passed' }
         end
 
+        def executed_interactions_count
+          examples_for_pact_uri
+            .collect { |e| e[:pact_interaction].object_id }
+            .uniq
+            .count
+        end
+
+        def all_interactions_count
+          pact_source.pact_hash['interactions'].count
+        end
+
         def test_results_hash_for_pact_uri
           {
-            examples: examples_for_pact_uri,
+            examples: examples_for_pact_uri.collect{ |e| clean_example(e) },
             summary: {
               exampleCount: examples_for_pact_uri.size,
               failureCount: count_failures_for_pact_uri
@@ -48,7 +61,7 @@ def test_results_hash_for_pact_uri
         end
 
         def clean_example(example)
-          example.reject{ |k, v| k == :pact_uri }
+          example.reject{ |k, v| k == :pact_uri || k == :pact_interaction }
         end
 
         attr_reader :pact_source, :test_results_hash

lib/pact/provider/verification_results/publish.rb

@@ -22,25 +22,29 @@ def initialize pact_source, verification_result
         end
 
         def call
-          if Pact.configuration.provider.publish_verification_results?
-            if Pact.configuration.provider.tags.any?
-              if tag_url('')
-                tag_versions
-              else
-                Pact.configuration.error_stream.puts "WARN: Cannot tag provider version as there is no link named pb:tag-version in the pact JSON."
-              end
-            end
-
-            if publication_url
-              publish
-            else
-              Pact.configuration.error_stream.puts "WARN: Cannot publish verification for #{consumer_name} as there is no link named pb:publish-verification-results in the pact JSON. If you are using a pact broker, please upgrade to version 2.0.0 or later."
-            end
+          if can_publish_verification_results?
+            tag_versions_if_configured
+            publish_verification_results
           end
         end
 
         private
 
+        def can_publish_verification_results?
+          return false unless Pact.configuration.provider.publish_verification_results?
+
+          if publication_url.nil?
+            Pact.configuration.error_stream.puts "WARN: Cannot publish verification for #{consumer_name} as there is no link named pb:publish-verification-results in the pact JSON. If you are using a pact broker, please upgrade to version 2.0.0 or later."
+            return false
+          end
+
+          if !verification_result.publishable?
+            Pact.configuration.error_stream.puts "WARN: Cannot publish verification for #{consumer_name} as not all interactions have been verified. Re-run the verification without the filter parameters or environment variables to publish the verification."
+            return false
+          end
+          true
+        end
+
         def publication_url
           @publication_url ||= pact_source.pact_hash.fetch('_links', {}).fetch('pb:publish-verification-results', {})['href']
         end
@@ -50,6 +54,16 @@ def tag_url tag
           href ? href.gsub('{tag}', tag) : nil
         end
 
+        def tag_versions_if_configured
+          if Pact.configuration.provider.tags.any?
+            if tag_url('')
+              tag_versions
+            else
+              Pact.configuration.error_stream.puts "WARN: Cannot tag provider version as there is no link named pb:tag-version in the pact JSON."
+            end
+          end
+        end
+
         def tag_versions
           Pact.configuration.provider.tags.each do | tag |
             uri = URI(tag_url(tag))
@@ -71,7 +85,7 @@ def tag_versions
           end
         end
 
-        def publish
+        def publish_verification_results
           uri = URI(publication_url)
           request = build_request('Post', uri, verification_result.to_json, "Publishing verification result #{verification_result} to")
           response = nil

lib/pact/provider/verification_results/publish_all.rb

@@ -15,10 +15,9 @@ def initialize pact_sources, test_results_hash
           @test_results_hash = test_results_hash
         end
 
-        # TODO do not publish unless all interactions have been run
         def call
-          verification_results.collect do | pair |
-            Publish.call(pair.first, pair.last)
+          verification_results.collect do | (pact_source, verification_result) |
+            Publish.call(pact_source, verification_result)
           end
         end
 

lib/pact/provider/verification_results/verification_result.rb

@@ -5,12 +5,17 @@ module Provider
     module VerificationResults
       class VerificationResult
 
-        def initialize success, provider_application_version, test_results_hash
+        def initialize publishable, success, provider_application_version, test_results_hash
+          @publishable = publishable
           @success = success
           @provider_application_version = provider_application_version
           @test_results_hash = test_results_hash
         end
 
+        def publishable?
+          @publishable
+        end
+
         def provider_application_version_set?
           !!provider_application_version
         end

spec/integration/publish_verification_spec.rb

@@ -25,6 +25,7 @@
 
   let(:pact_hash) do
     {
+      'interactions' => [{}],
       '_links' => {
         'pb:publish-verification-results' => {
           'href' => 'http://publish/'

spec/lib/pact/provider/verification_results/create_spec.rb

@@ -0,0 +1,113 @@
+require 'pact/provider/verification_results/create'
+
+module Pact
+  module Provider
+    module VerificationResults
+      describe Create do
+        before do
+          allow(Pact.configuration).to receive(:provider).and_return(provider_configuration)
+          allow(VerificationResult).to receive(:new).and_return(verification_result)
+        end
+
+        let(:verification_result) { double('VerificationResult') }
+        let(:provider_configuration) do
+          double('provider_configuration', application_version: '1.2.3')
+        end
+        let(:pact_source_1) do
+          instance_double('Pact::Provider::PactSource', uri: pact_uri_1, pact_hash: pact_hash_1)
+        end
+        let(:pact_uri_1) { instance_double('Pact::Provider::PactURI', uri: URI('foo')) }
+        let(:pact_uri_2) { instance_double('Pact::Provider::PactURI', uri: URI('bar')) }
+        let(:example_1) do
+          {
+            pact_uri: pact_uri_1,
+            pact_interaction: double('interaction'),
+            status: 'passed'
+          }
+        end
+        let(:example_2) do
+          {
+            pact_uri: pact_uri_2,
+            pact_interaction: double('interaction'),
+            status: 'passed'
+          }
+        end
+        let(:test_results_hash) do
+          {
+            examples: [example_1, example_2]
+          }
+        end
+        let(:pact_hash_1) do
+          {
+            'interactions' => [{}]
+          }
+        end
+
+        subject { Create.call(pact_source_1, test_results_hash) }
+
+        it "returns a verification result" do
+          expect(subject).to eq verification_result
+        end
+
+        it "creates a VerificationResult with the relevant test results" do
+          expected_test_results_hash = {
+            examples: [{ status: "passed" }],
+            summary: { exampleCount: 1, failureCount: 0}
+          }
+          expect(VerificationResult).to receive(:new).with(anything, anything, anything, expected_test_results_hash)
+          subject
+        end
+
+        it "creates a VerificationResult with the provider application version" do
+          expect(provider_configuration).to receive(:application_version)
+          expect(VerificationResult).to receive(:new).with(anything, anything, '1.2.3', anything)
+          subject
+        end
+
+        context "when every interaction has been executed" do
+          it "sets publishable to true" do
+            expect(VerificationResult).to receive(:new).with(true, anything, anything, anything)
+            subject
+          end
+        end
+
+        context "when not every interaction has been executed" do
+          let(:pact_hash_1) do
+            {
+              'interactions' => [{}, {}]
+            }
+          end
+          it "sets publishable to false" do
+            expect(VerificationResult).to receive(:new).with(false, anything, anything, anything)
+            subject
+          end
+        end
+
+        context "when all the examples passed" do
+          it "sets the success to true" do
+            expect(VerificationResult).to receive(:new).with(anything, true, anything, anything)
+            subject
+          end
+        end
+
+        context "when not all the examples passed" do
+          before do
+            example_1[:status] = 'notpassed'
+          end
+
+          it "sets the success to false" do
+            expect(VerificationResult).to receive(:new).with(anything, false, anything, anything)
+            subject
+          end
+
+          it "sets the failureCount" do
+            expect(VerificationResult).to receive(:new) do | _, _, _, test_results_hash|
+              expect(test_results_hash[:summary][:failureCount]).to eq 1
+            end
+            subject
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/pact/provider/verification_results/publish_spec.rb

@@ -35,11 +35,13 @@ module VerificationResults
           let(:app_version_set) { false }
           let(:verification_json) { '{"foo": "bar"}' }
           let(:publish_verification_results) { false }
+          let(:publishable) { true }
           let(:tags) { [] }
           let(:verification) do
             instance_double("Pact::Verifications::Verification",
               to_json: verification_json,
-              provider_application_version_set?: app_version_set
+              provider_application_version_set?: app_version_set,
+              publishable?: publishable
             )
           end
 
@@ -74,6 +76,15 @@ module VerificationResults
                 expect(WebMock).to have_requested(:post, publish_verification_url).with(body: verification_json, headers: {'Content-Type' => 'application/json'})
               end
 
+              context "when the verification result is not publishable" do
+                let(:publishable) { false }
+
+                it "does not publish the verification" do
+                  subject
+                  expect(WebMock).to_not have_requested(:post, 'http://broker/verifications')
+                end
+              end
+
               context "with tags" do
                 let(:tags) { ['foo'] }