From f8f87b72d048aa7b82e10441132dbd50b21dbc97 Mon Sep 17 00:00:00 2001 From: Evan Goldenberg Date: Tue, 11 Apr 2023 04:26:05 -0700 Subject: [PATCH 1/4] MONGOID-5595 Forward explain options to driver (#5594) --- lib/mongoid/contextual/mongo.rb | 20 ++++++++++---------- spec/mongoid/contextual/mongo_spec.rb | 11 ++++++++++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/lib/mongoid/contextual/mongo.rb b/lib/mongoid/contextual/mongo.rb index 85e27b254..72ddb5d98 100644 --- a/lib/mongoid/contextual/mongo.rb +++ b/lib/mongoid/contextual/mongo.rb @@ -38,6 +38,16 @@ class Mongo # @attribute [r] view The Mongo collection view. attr_reader :view + # Run an explain on the criteria. + # + # @example Explain the criteria. + # Band.where(name: "Depeche Mode").explain + # + # @param [ Hash ] options customizable options (See Mongo::Collection::View::Explainable) + # + # @return [ Hash ] The explain result. + def_delegator :view, :explain + attr_reader :documents_loader # Get the number of documents matching the query. @@ -177,16 +187,6 @@ def exists?(id_or_conditions = :none) end end - # Run an explain on the criteria. - # - # @example Explain the criteria. - # Band.where(name: "Depeche Mode").explain - # - # @return [ Hash ] The explain result. - def explain - view.explain - end - # Execute the find and modify command, used for MongoDB's # $findAndModify. # diff --git a/spec/mongoid/contextual/mongo_spec.rb b/spec/mongoid/contextual/mongo_spec.rb index b6e0da709..cc10e90e7 100644 --- a/spec/mongoid/contextual/mongo_spec.rb +++ b/spec/mongoid/contextual/mongo_spec.rb @@ -1471,7 +1471,16 @@ end it "returns the criteria explain path" do - expect(context.explain).to_not be_empty + explain = context.explain + expect(explain).to_not be_empty + expect(explain.keys).to include("queryPlanner", "executionStats", "serverInfo") + end + + it "respects options passed to explain" do + explain = context.explain(verbosity: :query_planner) + expect(explain).to_not be_empty + expect(explain.keys).to include("queryPlanner", "serverInfo") + expect(explain.keys).not_to include("executionStats") end end From 344ee2db173316b3cdc73a7822f779e749d08d3e Mon Sep 17 00:00:00 2001 From: Dmitry Rybakov Date: Wed, 19 Apr 2023 09:31:48 +0200 Subject: [PATCH 2/4] MONGOID-5592 Allow key vault client configuration (#5602) --- lib/mongoid/clients/factory.rb | 41 ++++++++++++---- spec/integration/encryption_spec.rb | 4 +- spec/mongoid/clients/factory_spec.rb | 73 ++++++++++++++++++++-------- 3 files changed, 87 insertions(+), 31 deletions(-) diff --git a/lib/mongoid/clients/factory.rb b/lib/mongoid/clients/factory.rb index aedc96047..379c33a15 100644 --- a/lib/mongoid/clients/factory.rb +++ b/lib/mongoid/clients/factory.rb @@ -58,15 +58,8 @@ def create_client(configuration) database = config.delete(:database) hosts = config.delete(:hosts) opts = config.delete(:options) || {} - if opts[:auto_encryption_options] - if opts[:auto_encryption_options].key?(:schema_map) - Mongoid.logger.warn( - 'The :schema_map is configured in the :auto_encryption_options for the client;' + - ' encryption setting in Mongoid documents will be ignored.' - ) - else - opts[:auto_encryption_options][:schema_map] = Mongoid.config.encryption_schema_map(database) - end + if opts.key?(:auto_encryption_options) + opts[:auto_encryption_options] = build_auto_encryption_options(opts, database) end unless config.empty? default_logger.warn("Unknown config options detected: #{config}.") @@ -81,6 +74,36 @@ def create_client(configuration) end end + # Build auto encryption options for the client based on the options + # provided in the Mongoid client configuration and the encryption + # schema map for the database. + # + # @param [ Hash ] opts Options from the Mongoid client configuration. + # @param [ String ] database Database name to use for encryption schema map. + # + # @return [ Hash | nil ] Auto encryption options for the client. + # + # @api private + def build_auto_encryption_options(opts, database) + return nil unless opts[:auto_encryption_options] + + opts[:auto_encryption_options].dup.tap do |auto_encryption_options| + if auto_encryption_options.key?(:schema_map) + default_logger.warn( + 'The :schema_map is configured in the :auto_encryption_options for the client;' + + ' encryption setting in Mongoid documents will be ignored.' + ) + else + auto_encryption_options[:schema_map] = Mongoid.config.encryption_schema_map(database) + end + if auto_encryption_options.key?(:key_vault_client) + auto_encryption_options[:key_vault_client] = Mongoid::Clients.with_name( + auto_encryption_options[:key_vault_client] + ) + end + end + end + MONGOID_WRAPPING_LIBRARY = { name: 'Mongoid', version: VERSION, diff --git a/spec/integration/encryption_spec.rb b/spec/integration/encryption_spec.rb index 70fc58028..a25ba5d8a 100644 --- a/spec/integration/encryption_spec.rb +++ b/spec/integration/encryption_spec.rb @@ -10,11 +10,13 @@ let(:config) do { default: { hosts: SpecConfig.instance.addresses, database: database_id }, + key_vault: { hosts: SpecConfig.instance.addresses, database: :key_vault }, encrypted: { hosts: SpecConfig.instance.addresses, database: database_id, options: { auto_encryption_options: { + key_vault_client: :key_vault, kms_providers: kms_providers, key_vault_namespace: key_vault_namespace, extra_options: extra_options @@ -29,11 +31,11 @@ end around do |example| - key_vault_client[key_vault_collection].drop Mongoid.default_client[Crypt::Patient.collection_name].drop existing_key_id = Crypt::Patient.encrypt_metadata[:key_id] Crypt::Patient.set_key_id(data_key_id) Mongoid::Config.send(:clients=, config) + Mongoid::Clients.with_name(:key_vault)[key_vault_collection].drop Crypt::Patient.store_in(client: :encrypted) example.run diff --git a/spec/mongoid/clients/factory_spec.rb b/spec/mongoid/clients/factory_spec.rb index 801a43c9b..37cbbf3dd 100644 --- a/spec/mongoid/clients/factory_spec.rb +++ b/spec/mongoid/clients/factory_spec.rb @@ -252,23 +252,6 @@ restore_config_clients include_context 'with encryption' - let(:config) do - { - default: { hosts: SpecConfig.instance.addresses, database: database_id }, - encrypted: { - hosts: SpecConfig.instance.addresses, - database: database_id, - options: { - auto_encryption_options: { - kms_providers: kms_providers, - key_vault_namespace: key_vault_namespace, - extra_options: extra_options - } - } - } - } - end - before do Mongoid::Config.send(:clients=, config) key_vault_client[key_vault_collection].drop @@ -282,12 +265,60 @@ described_class.create(:encrypted) end - it "returns a client" do - expect(client).to be_a(Mongo::Client) + context 'when no key vault client is provided' do + let(:config) do + { + default: { hosts: SpecConfig.instance.addresses, database: database_id }, + encrypted: { + hosts: SpecConfig.instance.addresses, + database: database_id, + options: { + auto_encryption_options: { + kms_providers: kms_providers, + key_vault_namespace: key_vault_namespace, + extra_options: extra_options + } + } + } + } + end + + it "returns a client" do + expect(client).to be_a(Mongo::Client) + end + + it 'sets schema_map for the client' do + expect(client.options[:auto_encryption_options][:schema_map]).not_to be_nil + end end - it 'sets schema_map for the client' do - expect(client.options[:auto_encryption_options][:schema_map]).not_to be_nil + context 'when a key vault client is provided' do + let(:config) do + { + default: { hosts: SpecConfig.instance.addresses, database: database_id }, + key_vault: { hosts: SpecConfig.instance.addresses, database: database_id }, + encrypted: { + hosts: SpecConfig.instance.addresses, + database: database_id, + options: { + auto_encryption_options: { + key_vault_client: :key_vault, + kms_providers: kms_providers, + key_vault_namespace: key_vault_namespace, + extra_options: extra_options + } + } + } + } + end + + it "returns a client" do + expect(client).to be_a(Mongo::Client) + end + + it 'sets key_vault_client option for the client' do + expect(client.options[:auto_encryption_options][:key_vault_client]).to eq(Mongoid::Clients.with_name(:key_vault)) + end end end end From 10c1fdb2d11087de47c36d2d9652bb500ae7706f Mon Sep 17 00:00:00 2001 From: johnnyshields Date: Sun, 23 Apr 2023 02:33:27 +0900 Subject: [PATCH 3/4] Auto-correct Rubocop --- lib/mongoid/clients/factory.rb | 4 ++-- spec/mongoid/clients/factory_spec.rb | 6 +++--- spec/mongoid/contextual/mongo_spec.rb | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/mongoid/clients/factory.rb b/lib/mongoid/clients/factory.rb index a8ec120b8..336e2ca82 100644 --- a/lib/mongoid/clients/factory.rb +++ b/lib/mongoid/clients/factory.rb @@ -96,8 +96,8 @@ def build_auto_encryption_options(opts, database) opts[:auto_encryption_options].dup.tap do |auto_encryption_options| if auto_encryption_options.key?(:schema_map) default_logger.warn( - 'The :schema_map is configured in the :auto_encryption_options for the client;' + - ' encryption setting in Mongoid documents will be ignored.' + 'The :schema_map is configured in the :auto_encryption_options for the client; ' \ + 'encryption setting in Mongoid documents will be ignored.' ) else auto_encryption_options[:schema_map] = Mongoid.config.encryption_schema_map(database) diff --git a/spec/mongoid/clients/factory_spec.rb b/spec/mongoid/clients/factory_spec.rb index 501d4974f..385515f08 100644 --- a/spec/mongoid/clients/factory_spec.rb +++ b/spec/mongoid/clients/factory_spec.rb @@ -280,12 +280,12 @@ } end - it "returns a client" do + it 'returns a client' do expect(client).to be_a(Mongo::Client) end it 'sets schema_map for the client' do - expect(client.options[:auto_encryption_options][:schema_map]).not_to be_nil + expect(client.options[:auto_encryption_options][:schema_map]).to_not be_nil end end @@ -309,7 +309,7 @@ } end - it "returns a client" do + it 'returns a client' do expect(client).to be_a(Mongo::Client) end diff --git a/spec/mongoid/contextual/mongo_spec.rb b/spec/mongoid/contextual/mongo_spec.rb index d700c5852..2a45774d7 100644 --- a/spec/mongoid/contextual/mongo_spec.rb +++ b/spec/mongoid/contextual/mongo_spec.rb @@ -1570,17 +1570,17 @@ described_class.new(criteria) end - it "returns the criteria explain path" do + it 'returns the criteria explain path' do explain = context.explain expect(explain).to_not be_empty - expect(explain.keys).to include("queryPlanner", "executionStats", "serverInfo") + expect(explain.keys).to include('queryPlanner', 'executionStats', 'serverInfo') end - it "respects options passed to explain" do + it 'respects options passed to explain' do explain = context.explain(verbosity: :query_planner) expect(explain).to_not be_empty - expect(explain.keys).to include("queryPlanner", "serverInfo") - expect(explain.keys).not_to include("executionStats") + expect(explain.keys).to include('queryPlanner', 'serverInfo') + expect(explain.keys).to_not include('executionStats') end end From a9def390da6f3e2bcd2bb2e99cad6a557dc4ada0 Mon Sep 17 00:00:00 2001 From: johnnyshields Date: Sun, 23 Apr 2023 03:15:08 +0900 Subject: [PATCH 4/4] Fix FLE tests --- .github/workflows/test.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c9c92ffc..7a03c6c33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,7 @@ jobs: ${{matrix.gemfile || 'Gemfile'}} mongodb-${{matrix.mongodb || '6.0'}} ${{matrix.topology || 'server'}} - ${{matrix.fle && 'FLE' || ''}} + ${{matrix.fle && 'fle=' || ''}}${{matrix.fle || ''}} ${{matrix.os || 'ubuntu-20.04'}}" env: CI: true @@ -107,27 +107,26 @@ jobs: topology: server # Field-Level Encryption + # TODO: support LIBMONGOCRYPT via path - ruby: ruby-3.2 gemfile: gemfiles/rails_7.0.gemfile mongodb: '6.0' topology: sharded_cluster - fle: true + fle: helper - ruby: ruby-3.1 gemfile: gemfiles/rails_6.1.gemfile mongodb: '6.0' topology: replica_set - fle: true + fle: helper - ruby: ruby-2.7 gemfile: gemfiles/rails_6.0.gemfile mongodb: '6.0' topology: server - fle: true + fle: helper steps: - name: repo checkout - uses: actions/checkout@v2 - with: - submodules: recursive + uses: actions/checkout@v3 - id: start-mongodb name: start mongodb uses: mongodb-labs/drivers-evergreen-tools@master @@ -138,6 +137,7 @@ jobs: uses: ruby/setup-ruby@v1 env: BUNDLE_GEMFILE: "${{matrix.gemfile || 'Gemfile'}}" + FLE: "${{matrix.fle || ''}}" with: ruby-version: "${{matrix.ruby}}" bundler: 2 @@ -145,11 +145,12 @@ jobs: run: bundle install --jobs 4 --retry 3 env: BUNDLE_GEMFILE: "${{matrix.gemfile || 'Gemfile'}}" + FLE: "${{matrix.fle || ''}}" - name: test timeout-minutes: 60 continue-on-error: "${{matrix.experimental || false}}" run: bundle exec rake spec env: BUNDLE_GEMFILE: "${{matrix.gemfile || 'Gemfile'}}" + FLE: "${{matrix.fle || ''}}" MONGODB_URI: "${{steps.start-mongodb.outputs.cluster-uri}}" - FLE: "${{matrix.fle || false}}"