Merge pull request #3071 from newrelic/lol-cat

Add HA CAT

test/multiverse/lib/multiverse/runner.rb

@@ -105,6 +105,7 @@ def execute_suites(filter, opts)
       'frameworks' => %w[grape padrino roda sinatra],
       'httpclients' => %w[async_http curb excon httpclient],
       'httpclients_2' => %w[typhoeus net_http httprb ethon httpx],
+      'hybrid_agent' => %w[hybrid_agent],
       'rails' => %w[active_support_broadcast_logger active_support_logger rails rails_prepend activemerchant],
 
       # these need services running in github actions, so they are separated

test/multiverse/suites/hybrid_agent/Envfile

@@ -0,0 +1,14 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+# instrumentation_methods :chain, :prepend
+
+suite_condition('OpenTelemetry requires CRuby version 3.1+') do
+  RUBY_VERSION >= '3.1.0'
+end
+
+gemfile <<~RB
+  gem 'opentelemetry-api'
+  gem 'opentelemetry-sdk'
+RB

test/multiverse/suites/hybrid_agent/README.md

@@ -0,0 +1,86 @@
+# Hybrid Agent Cross Agent Tests
+
+These tests are drawn from the [Hybrid Agent Cross Agent Test Example][example] repository.
+
+The file used to generate the test content can be found in [test/fixtures/cross_agent_tests/hybrid_agent.json][fixture]
+
+The parser converts most keys to snake case to adhere with Ruby's standard naming practices.
+
+## Updating the tests
+
+The test fixture is not currently in the cross agent tests repo and will need to be manually updated until it is moved.
+You can update the fixture by copying the [TestCaseDefinitions.json][test-cases] file and overwriting the
+[hybrid-agent.json] file in our test fixtures.
+
+If the fixture adds [a command][example-commands] that isn't already defined, add it to the `Commands` module.
+
+If the fixture adds [an assertion parameter][example-rules], please update the `AssertionParameters` module.
+
+The tests should fail with a helpful error message if either of these cases occur. Both the `Commands` and
+`AssertionParameters` modules are included in the `HybridAgentTest`
+
+At the time of their writing, the Ruby agent has not implemented Hybrid Agent functionality. The tests will fail if they
+are run.
+
+They have their own group in the [Multiverse::Runner module][runner], but it is not part of the CI, so it should not
+execute.
+
+At this time, only the OpenTelemetry::API methods are called. There is not an active SDK with a functional
+TracerProvider running. This may need to change once we implement the Hybrid Agent functionality.
+
+## Debugging
+
+### focus_tests
+
+The `HybridAgentTest` class includes a method, `#focus_tests` that can be used to run select tests based on the snake-
+cased version of the corresponding `testDescription` key in the `hybrid_agent.json` fixture.
+
+Example:
+```ruby
+# Update the focus_tests method with one or more testDescriptions you want to run
+def focus_tests
+  %w[does_not_create_segment_without_a_transaction]
+end
+
+```output
+$ bermq hybrid_agent
+
+...
+# Running:
+
+S.SSSSS
+
+Fabulous run in 0.003526s, 1985.2524 runs/s, 1701.6449 assertions/s.
+
+7 runs, 6 assertions, 0 failures, 0 errors, 6 skips
+```
+
+### ENABLE_OUTPUT
+
+You can pass `ENABLE_OUTPUT=true` to the test to get a print out of all the evaluated JSON content. This can be helpful
+to debug whether all levels of the test fixture are executed.
+
+Example:
+```shell
+$ ENABLE_OUTPUT=true bermq hybrid_agent
+
+TEST: does_not_create_segment_without_a_transaction
+do_work_in_span
+{span_name: "Bar", span_kind: :internal}
+The OpenTelmetry span should not be created
+{"operator" => "NotValid", "parameters" => {"object" => "currentOTelSpan"}}
+current_otel_span
+There should be no transaction
+{"operator" => "NotValid", "parameters" => {"object" => "currentTransaction"}}
+current_transaction
+Agent Output: transactions: []
+Agent Output: spans: []
+...
+```
+
+[example]: https://github.com/nrcventura/HybridAgentCrossAgentTestExample
+[example-commands]: https://github.com/nrcventura/HybridAgentCrossAgentTestExample/blob/main/README.md#commands
+[example-rules]: https://github.com/nrcventura/HybridAgentCrossAgentTestExample/blob/main/README.md#rules
+[test-cases]: https://github.com/nrcventura/HybridAgentCrossAgentTestExample/blob/main/TestCaseDefinitions.json
+[fixture]: ../../../fixtures/cross_agent_tests/hybrid_agent.json
+[runner]: ../../lib/multiverse/runner.rb

test/multiverse/suites/hybrid_agent/assertion_parameters.rb

@@ -0,0 +1,23 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module AssertionParameters
+  def current_otel_span
+    return nil if OpenTelemetry::Trace.current_span == OpenTelemetry::Trace::Span::INVALID
+
+    OpenTelemetry::Trace.current_span
+  end
+
+  def current_otel_span_context
+    current_otel_span&.context
+  end
+
+  def current_transaction
+    NewRelic::Agent::Tracer.current_transaction
+  end
+
+  def injected
+    # TODO
+  end
+end

test/multiverse/suites/hybrid_agent/commands.rb

@@ -0,0 +1,70 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module Commands
+  def do_work_in_span(span_name:, span_kind:, &block)
+    span = @tracer.start_span(span_name, kind: span_kind)
+    yield if block
+  ensure
+    span&.finish
+  end
+
+  def do_work_in_span_with_remote_parent(span_name:, span_kind:, &block)
+    span = @tracer.start_span(span_name, kind: span_kind)
+    span.context.instance_variable_set(:@remote, true)
+    yield if block
+  ensure
+    span&.finish
+  end
+
+  def do_work_in_span_with_inbound_context(span_name:, span_kind:, trace_id_in_header:,
+    span_id_in_header:, sampled_flag_in_header:, &block)
+    # TODO
+    yield if block
+  end
+
+  def do_work_in_transaction(transaction_name:, &block)
+    transaction = NewRelic::Agent::Tracer.start_transaction(name: transaction_name, category: :web)
+    yield if block
+  ensure
+    transaction&.finish
+  end
+
+  def do_work_in_segment(segment_name:, &block)
+    segment = NewRelic::Agent::Tracer.start_segment(name: segment_name)
+    yield if block
+  ensure
+    segment&.finish
+  end
+
+  def add_otel_attribute(name:, value:, &block)
+    OpenTelemetry::Trace.current_span&.set_attribute(name, value)
+    yield if block
+  end
+
+  def record_exception_on_span(error_message:, &block)
+    exception = StandardError.new(error_message)
+    OpenTelemetry::Trace.current_span.record_exception(exception)
+    yield if block
+  end
+
+  def simulate_external_call(url:, &block)
+    # TODO
+    yield if block
+  end
+
+  def o_tel_inject_headers(&block)
+    # TODO: figure out how to pass the request
+    request = {}
+    OpenTelemetry.propagation.inject(request)
+    yield if block
+  end
+
+  def nr_inject_headers(&block)
+    # TODO: figure out how to get the headers
+    headers = {}
+    NewRelic::Agent::DistributedTracing.insert_distributed_trace_headers(headers)
+    yield if block
+  end
+end

test/multiverse/suites/hybrid_agent/config/newrelic.yml

@@ -0,0 +1,17 @@
+---
+development:
+  error_collector:
+    enabled: true
+  apdex_t: 0.5
+  monitor_mode: true
+  license_key: bootstrap_newrelic_admin_license_key_000
+  app_name: test
+  log_level: debug
+  host: 127.0.0.1
+  api_host: 127.0.0.1
+  transaction_trace:
+    record_sql: obfuscated
+    enabled: true
+    stack_trace_threshold: 0.5
+    transaction_threshold: 1.0
+  capture_params: false

test/multiverse/suites/hybrid_agent/hybrid_agent_test.rb

@@ -0,0 +1,48 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require 'opentelemetry'
+require_relative 'commands'
+require_relative 'assertion_parameters'
+require_relative 'parsing_helpers'
+
+class HybridAgentTest < Minitest::Test
+  include Commands
+  include AssertionParameters
+  include ParsingHelpers
+
+  def setup
+    @tracer = OpenTelemetry.tracer_provider.tracer
+  end
+
+  # This method, when returning a non-empty array, will cause the tests defined in the
+  # JSON file to be skipped if they're not listed here. Useful for focusing on specific
+  # failing tests.
+  # It looks for the snake cased version of the testDescription field in the JSON
+  # Ex: %w[does_not_create_segment_without_a_transaction] would only run
+  # `"testDescription": "Does not create segment without a transaction"`
+  def focus_tests
+    %w[]
+  end
+
+  test_cases = load_cross_agent_test('hybrid_agent')
+  test_cases.each do |test_case|
+    name = test_case['testDescription'].downcase.tr(' ', '_')
+
+    define_method("test_hybrid_agent_#{name}") do
+      if focus_tests.empty? || focus_tests.include?(name)
+        puts "TEST: #{name}" if ENV['ENABLE_OUTPUT']
+
+        operations = test_case['operations']
+        operations.map do |o|
+          parse_operation(o)
+        end
+
+        harvest_and_verify_agent_output(test_case['agentOutput'])
+      else
+        skip('marked pending by exclusion from #focus_tests')
+      end
+    end
+  end
+end