Properly test FLE with crypt_shared

.evergreen/config.yml

@@ -1063,6 +1063,10 @@ axes:
         display_name: via LMC path
         variables:
           FLE: path
+      - id: "mongocryptd"
+        display_name: mongocryptd (no crypt_shared)
+        variables:
+          FLE: mongocryptd
 
   - id: ocsp-algorithm
     display_name: OCSP Algorithm
@@ -1532,6 +1536,19 @@ buildvariants:
     tasks:
       - name: "test-fle"
 
+  - matrix_name: "fle-mongocryptd"
+    matrix_spec:
+      auth-and-ssl: "noauth-and-nossl"
+      ruby: "ruby-4.0"
+      topology: [replica-set, sharded-cluster]
+      mongodb-version: "8.0"
+      os: ubuntu2204
+      fle: mongocryptd
+    display_name: "FLE mongocryptd: ${mongodb-version} ${topology} ${ruby}"
+    tags: ["pr"]
+    tasks:
+      - name: "test-fle"
+
   - matrix_name: "kerberos-unit"
     matrix_spec:
       ruby: "ruby-4.0"

.evergreen/config/axes.yml.erb

@@ -288,6 +288,10 @@ axes:
         display_name: via LMC path
         variables:
           FLE: path
+      - id: "mongocryptd"
+        display_name: mongocryptd (no crypt_shared)
+        variables:
+          FLE: mongocryptd
 
   - id: ocsp-algorithm
     display_name: OCSP Algorithm

.evergreen/config/standard.yml.erb

@@ -355,6 +355,19 @@ buildvariants:
     tasks:
       - name: "test-fle"
 
+  - matrix_name: "fle-mongocryptd"
+    matrix_spec:
+      auth-and-ssl: "noauth-and-nossl"
+      ruby: <%= latest_ruby %>
+      topology: [replica-set, sharded-cluster]
+      mongodb-version: <%= latest_stable_mdb %>
+      os: ubuntu2204
+      fle: mongocryptd
+    display_name: "FLE mongocryptd: ${mongodb-version} ${topology} ${ruby}"
+    tags: ["pr"]
+    tasks:
+      - name: "test-fle"
+
   - matrix_name: "kerberos-unit"
     matrix_spec:
       ruby: <%= latest_ruby %>

.evergreen/run-tests.sh

@@ -65,7 +65,7 @@ fi
 
 # Make sure cmake is installed (in case we need to install the libmongocrypt
 # helper)
-if [ "$FLE" = "helper" ]; then
+if [ -n "$FLE" ]; then
   install_cmake
 fi
 
@@ -193,23 +193,25 @@ elif test "$AUTH" = kerberos; then
 fi
 
 if test -n "$FLE"; then
-  # Downloading crypt shared lib
-  if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then
-    crypt_shared_version=${CRYPT_SHARED_VERSION:-$("${BINDIR}"/mongod --version | grep -oP 'db version v\K.*')}
-    python3 -u .evergreen/mongodl.py --component crypt_shared -V ${crypt_shared_version} --out $(pwd)/csfle_lib  --target $(host_distro) || true
-    if test -f $(pwd)/csfle_lib/lib/mongo_crypt_v1.so
-    then
-      export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/csfle_lib/lib/mongo_crypt_v1.so
+  # Downloading crypt shared lib (skipped for mongocryptd-only configuration)
+  if test "$FLE" != "mongocryptd"; then
+    if [ -z "$MONGO_CRYPT_SHARED_DOWNLOAD_URL" ]; then
+      crypt_shared_version=${CRYPT_SHARED_VERSION:-$("${BINDIR}"/mongod --version | grep -oP 'db version v\K.*')}
+      python3 -u .evergreen/mongodl.py --component crypt_shared -V ${crypt_shared_version} --out $(pwd)/csfle_lib  --target $(host_distro) || true
+      if test -f $(pwd)/csfle_lib/lib/mongo_crypt_v1.so
+      then
+        export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/csfle_lib/lib/mongo_crypt_v1.so
+      else
+        echo 'Could not find crypt_shared library'
+      fi
     else
-      echo 'Could not find crypt_shared library'
+      echo "Downloading crypt_shared package from $MONGO_CRYPT_SHARED_DOWNLOAD_URL"
+      mkdir -p $(pwd)/csfle_lib
+      cd $(pwd)/csfle_lib
+      curl --retry 3 -fL $MONGO_CRYPT_SHARED_DOWNLOAD_URL | tar zxf -
+      export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/lib/mongo_crypt_v1.so
+      cd -
     fi
-  else
-    echo "Downloading crypt_shared package from $MONGO_CRYPT_SHARED_DOWNLOAD_URL"
-    mkdir -p $(pwd)/csfle_lib
-    cd $(pwd)/csfle_lib
-    curl --retry 3 -fL $MONGO_CRYPT_SHARED_DOWNLOAD_URL | tar zxf -
-    export MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH=$(pwd)/lib/mongo_crypt_v1.so
-    cd -
   fi
 
   # Start the KMS servers first so that they are launching while we are

gemfiles/standard.rb

@@ -69,6 +69,6 @@ def standard_dependencies
     gem 'ruby-lsp', platforms: :mri
   end
 
-  gem 'libmongocrypt-helper', '~> 1.14.0' if ENV['FLE'] == 'helper'
+  gem 'libmongocrypt-helper', '~> 1.14.0' if %w[helper mongocryptd].include?(ENV['FLE'])
 end
 # rubocop:enable Metrics/AbcSize, Metrics/MethodLength, Metrics/BlockLength

lib/mongo/crypt/auto_encrypter.rb

@@ -105,6 +105,7 @@ def initialize(options)
           bypass_query_analysis: @options[:bypass_query_analysis],
           crypt_shared_lib_path: @options[:extra_options][:crypt_shared_lib_path],
           crypt_shared_lib_required: @options[:extra_options][:crypt_shared_lib_required],
+          disable_crypt_shared_lib_search: @options[:extra_options][:disable_crypt_shared_lib_search],
         )
 
         @mongocryptd_options = @options[:extra_options].slice(

lib/mongo/crypt/handle.rb

@@ -62,6 +62,11 @@ class Handle
       # @option options [ Boolean | nil ] :explicit_encryption_only Whether this
       #   handle is going to be used only for explicit encryption. If true,
       #   libmongocrypt is instructed not to load crypt shared library.
+      # @option options [ Boolean | nil ] :disable_crypt_shared_lib_search When
+      #   true, suppresses the automatic "$SYSTEM" search for crypt_shared. Use
+      #   this when a previous Handle in the same process has already loaded the
+      #   library via a path override and you want to avoid the conflicting-load
+      #   error that libmongocrypt raises on a subsequent "$SYSTEM" search.
       # @option options [ Logger ] :logger A Logger object to which libmongocrypt logs
       #   will be sent
       def initialize(kms_providers, kms_tls_options, options={})
@@ -85,9 +90,10 @@ def initialize(kms_providers, kms_tls_options, options={})
 
         @crypt_shared_lib_path = options[:crypt_shared_lib_path]
         @explicit_encryption_only = options[:explicit_encryption_only]
+        @disable_crypt_shared_lib_search = options[:disable_crypt_shared_lib_search]
         if @crypt_shared_lib_path
           Binding.setopt_set_crypt_shared_lib_path_override(self, @crypt_shared_lib_path)
-        elsif !@bypass_query_analysis && !@explicit_encryption_only
+        elsif !@bypass_query_analysis && !@explicit_encryption_only && !@disable_crypt_shared_lib_search
           Binding.setopt_append_crypt_shared_lib_search_path(self, "$SYSTEM")
         end
 

spec/integration/client_side_encryption/auto_encryption_mongocryptd_spawn_spec.rb

@@ -21,7 +21,11 @@
             schema_map: { 'auto_encryption.users' => schema_map },
             extra_options: {
               mongocryptd_spawn_path: 'echo hello world',
-              mongocryptd_spawn_args: []
+              mongocryptd_spawn_args: [],
+              # Suppress $SYSTEM crypt_shared search to avoid "existing library"
+              # conflicts on macOS when another spec in the same process has
+              # already loaded crypt_shared via an explicit path override.
+              disable_crypt_shared_lib_search: true,
             }
           },
           database: 'auto_encryption'

spec/integration/client_side_encryption/auto_encryption_reconnect_spec.rb

@@ -11,6 +11,14 @@
   # actually require a clean slate. https://jira.mongodb.org/browse/RUBY-2138
   clean_slate
 
+  # This spec tests reconnection with mongocryptd as the encryption backend.
+  # It directly manipulates mongocryptd_client, which is only created when
+  # crypt_shared is not available. Force the mongocryptd path regardless of
+  # whether MONGO_RUBY_DRIVER_CRYPT_SHARED_LIB_PATH is set.
+  around do |example|
+    SpecConfig.instance.without_crypt_shared_lib_path { example.run }
+  end
+
   include_context 'define shared FLE helpers'
 
   let(:client) do

spec/integration/client_side_encryption/bypass_mongocryptd_spawn_spec.rb

@@ -31,6 +31,7 @@
                 mongocryptd_bypass_spawn: true,
                 mongocryptd_uri: "mongodb://localhost:#{mongocryptd_port}/db?serverSelectionTimeoutMS=1000",
                 mongocryptd_spawn_args: [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=#{mongocryptd_port}"],
+                disable_crypt_shared_lib_search: true,
               },
             },
             database: 'db'
@@ -56,6 +57,7 @@
               bypass_auto_encryption: true,
               extra_options: {
                 mongocryptd_spawn_args: [ "--pidfilepath=bypass-spawning-mongocryptd.pid", "--port=#{mongocryptd_port}"],
+                disable_crypt_shared_lib_search: true,
               },
             },
             database: 'db'

spec/integration/client_side_encryption/data_key_spec.rb

@@ -171,7 +171,13 @@
 
         expect do
           client_encrypted['coll'].insert_one(encrypted_placeholder: encrypted)
-        end.to raise_error(Mongo::Error::OperationFailure, /Cannot encrypt element of type(: encrypted binary data| binData)/)
+        # With mongocryptd the error comes back as OperationFailure from the
+        # markForEncryption command response; with crypt_shared it is raised
+        # directly by libmongocrypt as a CryptError. Both are Mongo::Error subclasses.
+        end.to raise_error { |error|
+          expect(error).to be_a(Mongo::Error::OperationFailure).or be_a(Mongo::Error::CryptError)
+          expect(error.message).to match(/Cannot encrypt element of type(: encrypted binary data| binData)/)
+        }
       end
     end
 

spec/mongo/client_construction_spec.rb

@@ -177,12 +177,20 @@
         let(:bypass_auto_encryption) { true }
 
         let(:extra_options) do
-          {
+          opts = {
             mongocryptd_uri: mongocryptd_uri,
             mongocryptd_bypass_spawn: mongocryptd_bypass_spawn,
             mongocryptd_spawn_path: mongocryptd_spawn_path,
             mongocryptd_spawn_args: mongocryptd_spawn_args,
           }
+          # Use the explicit path when available so every Handle in the process
+          # loads via the same mechanism, avoiding the "An existing crypt_shared
+          # library is loaded" conflict on macOS when specs run in the same process.
+          if SpecConfig.instance.crypt_shared_lib_path
+            opts[:crypt_shared_lib_path] =
+              SpecConfig.instance.crypt_shared_lib_path
+          end
+          opts
         end
 
         let(:mongocryptd_uri) { 'mongodb://localhost:27021' }
@@ -298,17 +306,34 @@
               expect(encryption_options[:extra_options][:mongocryptd_bypass_spawn]).to eq(mongocryptd_bypass_spawn)
               expect(encryption_options[:extra_options][:mongocryptd_spawn_path]).to eq(mongocryptd_spawn_path)
               expect(encryption_options[:extra_options][:mongocryptd_spawn_args]).to eq(mongocryptd_spawn_args)
+            end
 
-              expect(client.encrypter.mongocryptd_client.options[:monitoring_io]).to be false
+            context 'without crypt_shared library' do
+              let(:extra_options) do
+                super().reject { |k, _| k == :crypt_shared_lib_path }.merge(disable_crypt_shared_lib_search: true)
+              end
+
+              it 'creates mongocryptd_client with monitoring_io: false' do
+                expect(client.encrypter.mongocryptd_client.options[:monitoring_io]).to be false
+              end
             end
 
             context 'with default extra options' do
               let(:auto_encryption_options) do
-                {
+                opts = {
                   key_vault_namespace: key_vault_namespace,
                   kms_providers: kms_providers,
                   schema_map: schema_map,
                 }
+                # When the env var is set, pass the explicit crypt_shared path so
+                # that Handle in this process uses the same load mechanism as any
+                # prior Handle, avoiding the "existing library" conflict on macOS.
+                # The test assertions only check mongocryptd default values, so this
+                # does not affect what is being verified.
+                if (path = SpecConfig.instance.crypt_shared_lib_path)
+                  opts[:extra_options] = { crypt_shared_lib_path: path }
+                end
+                opts
               end
 
               it 'sets key_vault_client with no encryption options' do
@@ -1998,7 +2023,7 @@
               block_client.encrypter.mongocryptd_client,
               block_client.encrypter.key_vault_client,
               block_client.encrypter.metadata_client
-            ].each do |crypt_client|
+            ].compact.each do |crypt_client|
               expect(crypt_client.cluster.connected?).to be false
             end
           end

spec/mongo/crypt/auto_encrypter_spec.rb

@@ -419,6 +419,10 @@
   context 'when using crypt shared library' do
     min_server_version '6.0.0'
 
+    before do
+      skip 'Requires crypt_shared library' unless SpecConfig.instance.crypt_shared_lib_path
+    end
+
     let(:auto_encrypter_extra_options) do
       {
         crypt_shared_lib_path: SpecConfig.instance.crypt_shared_lib_path

spec/mongo/crypt/handle_spec.rb

@@ -97,6 +97,12 @@
       context 'with crypt_shared_lib_path' do
         min_server_version '6.0.0'
 
+        before(:all) do
+          if ENV['FLE'] == 'mongocryptd'
+            skip 'FLE=mongocryptd is incompatible with unloaded binding tests'
+          end
+        end
+
         context 'with correct path' do
           let(:crypt_shared_lib_path) do
             SpecConfig.instance.crypt_shared_lib_path
@@ -121,6 +127,12 @@
       context 'with crypt_shared_lib_required' do
         min_server_version '6.0.0'
 
+        before(:all) do
+          if ENV['FLE'] == 'mongocryptd'
+            skip 'FLE=mongocryptd is incompatible with unloaded binding tests'
+          end
+        end
+
         context 'set to true' do
           let(:crypt_shared_lib_required) do
             true

spec/support/crypt.rb

@@ -96,10 +96,22 @@ module Crypt
     end
 
     let(:extra_options) do
-      {
+      opts = {
         mongocryptd_spawn_args: ["--port=#{SpecConfig.instance.mongocryptd_port}"],
         mongocryptd_uri: "mongodb://localhost:#{SpecConfig.instance.mongocryptd_port}",
       }
+      if SpecConfig.instance.crypt_shared_lib_path
+        # Always use the explicit path when available so that every Handle in
+        # the process uses the same load mechanism and libmongocrypt does not
+        # raise "An existing crypt_shared library is loaded" errors.
+        opts[:crypt_shared_lib_path] = SpecConfig.instance.crypt_shared_lib_path
+      elsif SpecConfig.instance.suppress_crypt_shared_lib_search?
+        # Inside without_crypt_shared_lib_path: skip the "$SYSTEM" search
+        # entirely so we don't conflict with any library already loaded by a
+        # previous Handle via a path override.
+        opts[:disable_crypt_shared_lib_search] = true
+      end
+      opts
     end
 
     let(:kms_tls_options) do

spec/support/spec_config.rb

@@ -519,6 +519,15 @@ def without_crypt_shared_lib_path
     @without_crypt_shared_lib_path = saved
   end
 
+  # Returns true when inside a without_crypt_shared_lib_path block. Used by
+  # test helpers to set disable_crypt_shared_lib_search on the Handle, which
+  # prevents the "$SYSTEM" search and avoids the "An existing crypt_shared
+  # library is loaded" error that occurs when a previous Handle in the same
+  # process loaded the library via a path override.
+  def suppress_crypt_shared_lib_search?
+    !!@without_crypt_shared_lib_path
+  end
+
   attr_accessor :crypt_shared_lib_required
 
   def require_crypt_shared

spec/support/utils.rb

@@ -672,6 +672,17 @@ def convert_auto_encryption_extra_options(opts)
       extra_options[:crypt_shared_lib_required] = SpecConfig.instance.crypt_shared_lib_required
       extra_options[:crypt_shared_lib_path] = SpecConfig.instance.crypt_shared_lib_path
       extra_options[:mongocryptd_uri] = 'mongodb://localhost:27777'
+    elsif !opts[:bypass_query_analysis]
+      if SpecConfig.instance.crypt_shared_lib_path
+        # Always use the explicit path so every Handle in the process loads via
+        # the same mechanism; prevents the "An existing crypt_shared library is
+        # loaded" conflict when mixing path-override and $SYSTEM Handles.
+        extra_options[:crypt_shared_lib_path] = SpecConfig.instance.crypt_shared_lib_path
+      elsif SpecConfig.instance.suppress_crypt_shared_lib_search?
+        # Inside without_crypt_shared_lib_path: suppress the "$SYSTEM" search
+        # to avoid conflicting with any library already resident in the process.
+        extra_options[:disable_crypt_shared_lib_search] = true
+      end
     end
 
     extra_options