Close vector search final-phase gaps with sample configs and queries

pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java

@@ -738,6 +738,38 @@ public static Integer getVectorEfSearch(Map<String, String> queryOptions) {
     return checkedParseIntPositive(QueryOptionKey.VECTOR_EF_SEARCH, efSearch);
   }
 
+  /**
+   * Returns whether HNSW should use relative-distance competitive checks, or {@code null} if not set.
+   */
+  @Nullable
+  public static Boolean getVectorUseRelativeDistance(Map<String, String> queryOptions) {
+    return checkedParseBooleanNullable(QueryOptionKey.VECTOR_USE_RELATIVE_DISTANCE,
+        queryOptions.get(QueryOptionKey.VECTOR_USE_RELATIVE_DISTANCE));
+  }
+
+  /**
+   * Returns whether HNSW should use a bounded collector queue, or {@code null} if not set.
+   */
+  @Nullable
+  public static Boolean getVectorUseBoundedQueue(Map<String, String> queryOptions) {
+    return checkedParseBooleanNullable(QueryOptionKey.VECTOR_USE_BOUNDED_QUEUE,
+        queryOptions.get(QueryOptionKey.VECTOR_USE_BOUNDED_QUEUE));
+  }
+
+  @Nullable
+  private static Boolean checkedParseBooleanNullable(String optionName, @Nullable String optionValue) {
+    if (optionValue == null) {
+      return null;
+    }
+    String normalized = optionValue.trim();
+    if ("true".equalsIgnoreCase(normalized)) {
+      return Boolean.TRUE;
+    }
+    if ("false".equalsIgnoreCase(normalized)) {
+      return Boolean.FALSE;
+    }
+    throw new IllegalArgumentException(optionName + " must be either true or false, got: " + optionValue);
+  }
 
   public static int getSortExchangeCopyThreshold(Map<String, String> options, int i) {
     String sortExchangeCopyThreshold = options.get(QueryOptionKey.SORT_EXCHANGE_COPY_THRESHOLD);

pinot-core/src/main/java/org/apache/pinot/core/operator/filter/ExactVectorScanFilterOperator.java

@@ -101,9 +101,11 @@ public ExactVectorScanFilterOperator(ForwardIndexReader<?> forwardIndexReader,
     _column = column;
     _hasDistanceThreshold = searchParams.hasDistanceThreshold();
     _distanceThreshold = searchParams.getDistanceThreshold();
+    float effectiveThreshold = _hasDistanceThreshold ? _distanceThreshold : -1f;
     _vectorExplainContext = new VectorExplainContext(VectorDistanceUtils.resolveBackendType(vectorIndexConfig),
         VectorDistanceUtils.resolveDistanceFunction(vectorIndexConfig), VectorExecutionMode.EXACT_SCAN,
-        VectorSearchParams.DEFAULT_NPROBE, false, predicate.getTopK(), fallbackReason);
+        VectorSearchParams.DEFAULT_NPROBE, false, predicate.getTopK(), fallbackReason, null, 0, effectiveThreshold,
+        VectorSearchMode.EXACT_SCAN, -1.0, null, null);
   }
 
   @Override

pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorExplainContext.java

@@ -45,96 +45,21 @@ public final class VectorExplainContext {
   private final float _effectiveThreshold;
   private final VectorSearchMode _vectorSearchMode;
   private final double _filterSelectivity;
+  @Nullable
+  private final Boolean _effectiveHnswUseRelativeDistance;
+  @Nullable
+  private final Boolean _effectiveHnswUseBoundedQueue;
 
   /**
-   * Backward-compatible constructor without the new fields.
-   */
-  public VectorExplainContext(VectorBackendType backendType,
-      VectorIndexConfig.VectorDistanceFunction distanceFunction, int effectiveNprobe, boolean effectiveExactRerank,
-      int effectiveSearchCount, @Nullable String fallbackReason) {
-    this(backendType, distanceFunction, null, effectiveNprobe, effectiveExactRerank, effectiveSearchCount,
-        fallbackReason, null, 0, -1f, VectorSearchMode.POST_FILTER_ANN, -1.0);
-  }
-
-  /**
-   * Constructor with execution mode but without the Phase 4 fields.
-   */
-  public VectorExplainContext(VectorBackendType backendType,
-      VectorIndexConfig.VectorDistanceFunction distanceFunction, @Nullable VectorExecutionMode executionMode,
-      int effectiveNprobe, boolean effectiveExactRerank, int effectiveSearchCount,
-      @Nullable String fallbackReason) {
-    this(backendType, distanceFunction, executionMode, effectiveNprobe, effectiveExactRerank, effectiveSearchCount,
-        fallbackReason, null, 0, -1f, VectorSearchMode.POST_FILTER_ANN, -1.0);
-  }
-
-  /**
-   * Full constructor with all fields including search mode, efSearch, and threshold.
-   *
-   * @param backendType the resolved backend type
-   * @param distanceFunction the distance function
-   * @param effectiveNprobe the effective nprobe value (0 for non-IVF backends)
-   * @param effectiveExactRerank whether exact rerank is effective
-   * @param effectiveSearchCount the effective search count
-   * @param fallbackReason reason for any fallback, or null
-   * @param searchMode the search mode label for adaptive planner, or null
-   * @param effectiveEfSearch the effective efSearch value (0 if not applicable)
-   * @param effectiveThreshold the effective distance threshold (-1 if not set)
-   */
-  public VectorExplainContext(VectorBackendType backendType,
-      VectorIndexConfig.VectorDistanceFunction distanceFunction, int effectiveNprobe, boolean effectiveExactRerank,
-      int effectiveSearchCount, @Nullable String fallbackReason, @Nullable String searchMode,
-      int effectiveEfSearch, float effectiveThreshold) {
-    this(backendType, distanceFunction, null, effectiveNprobe, effectiveExactRerank, effectiveSearchCount,
-        fallbackReason, searchMode, effectiveEfSearch, effectiveThreshold, VectorSearchMode.POST_FILTER_ANN,
-        -1.0);
-  }
-
-  /**
-   * Full constructor with all fields including vector search mode and filter selectivity.
-   *
-   * @param backendType the resolved backend type
-   * @param distanceFunction the distance function
-   * @param effectiveNprobe the effective nprobe value (0 for non-IVF backends)
-   * @param effectiveExactRerank whether exact rerank is effective
-   * @param effectiveSearchCount the effective search count
-   * @param fallbackReason reason for any fallback, or null
-   * @param searchMode the search mode label for adaptive planner, or null
-   * @param effectiveEfSearch the effective efSearch value (0 if not applicable)
-   * @param effectiveThreshold the effective distance threshold (-1 if not set)
-   * @param vectorSearchMode the vector search mode enum
-   * @param filterSelectivity the filter selectivity (0.0 to 1.0), or -1 if no filter
-   */
-  public VectorExplainContext(VectorBackendType backendType,
-      VectorIndexConfig.VectorDistanceFunction distanceFunction, int effectiveNprobe, boolean effectiveExactRerank,
-      int effectiveSearchCount, @Nullable String fallbackReason, @Nullable String searchMode,
-      int effectiveEfSearch, float effectiveThreshold, VectorSearchMode vectorSearchMode,
-      double filterSelectivity) {
-    this(backendType, distanceFunction, null, effectiveNprobe, effectiveExactRerank, effectiveSearchCount,
-        fallbackReason, searchMode, effectiveEfSearch, effectiveThreshold, vectorSearchMode, filterSelectivity);
-  }
-
-  /**
-   * Full constructor with all fields including execution mode and all Phase 4 fields.
-   *
-   * @param backendType the resolved backend type
-   * @param distanceFunction the distance function
-   * @param executionMode the execution mode, or null if not yet determined
-   * @param effectiveNprobe the effective nprobe value (0 for non-IVF backends)
-   * @param effectiveExactRerank whether exact rerank is effective
-   * @param effectiveSearchCount the effective search count
-   * @param fallbackReason reason for any fallback, or null
-   * @param searchMode the search mode label for adaptive planner, or null
-   * @param effectiveEfSearch the effective efSearch value (0 if not applicable)
-   * @param effectiveThreshold the effective distance threshold (-1 if not set)
-   * @param vectorSearchMode the vector search mode enum
-   * @param filterSelectivity the filter selectivity (0.0 to 1.0), or -1 if no filter
+   * Full constructor with all fields including HNSW runtime control metadata.
    */
   public VectorExplainContext(VectorBackendType backendType,
       VectorIndexConfig.VectorDistanceFunction distanceFunction, @Nullable VectorExecutionMode executionMode,
       int effectiveNprobe, boolean effectiveExactRerank, int effectiveSearchCount,
       @Nullable String fallbackReason, @Nullable String searchMode,
       int effectiveEfSearch, float effectiveThreshold, VectorSearchMode vectorSearchMode,
-      double filterSelectivity) {
+      double filterSelectivity, @Nullable Boolean effectiveHnswUseRelativeDistance,
+      @Nullable Boolean effectiveHnswUseBoundedQueue) {
     _backendType = backendType;
     _distanceFunction = distanceFunction;
     _executionMode = executionMode;
@@ -147,6 +72,8 @@ public VectorExplainContext(VectorBackendType backendType,
     _effectiveThreshold = effectiveThreshold;
     _vectorSearchMode = vectorSearchMode;
     _filterSelectivity = filterSelectivity;
+    _effectiveHnswUseRelativeDistance = effectiveHnswUseRelativeDistance;
+    _effectiveHnswUseBoundedQueue = effectiveHnswUseBoundedQueue;
   }
 
   public VectorBackendType getBackendType() {
@@ -218,4 +145,20 @@ public VectorSearchMode getVectorSearchMode() {
   public double getFilterSelectivity() {
     return _filterSelectivity;
   }
+
+  /**
+   * Returns the effective HNSW relative-distance check mode, or null if not applicable.
+   */
+  @Nullable
+  public Boolean getEffectiveHnswUseRelativeDistance() {
+    return _effectiveHnswUseRelativeDistance;
+  }
+
+  /**
+   * Returns the effective HNSW bounded queue mode, or null if not applicable.
+   */
+  @Nullable
+  public Boolean getEffectiveHnswUseBoundedQueue() {
+    return _effectiveHnswUseBoundedQueue;
+  }
 }

pinot-core/src/main/java/org/apache/pinot/core/operator/filter/VectorRadiusFilterOperator.java

@@ -30,6 +30,7 @@
 import org.apache.pinot.core.operator.docidsets.BitmapDocIdSet;
 import org.apache.pinot.segment.spi.index.creator.VectorBackendType;
 import org.apache.pinot.segment.spi.index.creator.VectorIndexConfig;
+import org.apache.pinot.segment.spi.index.reader.ApproximateRadiusVectorIndexReader;
 import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
 import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
 import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
@@ -72,6 +73,9 @@ public class VectorRadiusFilterOperator extends BaseFilterOperator {
   private final VectorBackendType _backendType;
   private final VectorIndexConfig.VectorDistanceFunction _distanceFunction;
   private final boolean _hasVectorIndex;
+  private final boolean _backendSupportsApproximateRadius;
+  private final boolean _readerSupportsApproximateRadius;
+  private final boolean _useApproximateRadiusPath;
   private ImmutableRoaringBitmap _matches;
 
   /**
@@ -95,6 +99,10 @@ public VectorRadiusFilterOperator(ForwardIndexReader<?> forwardIndexReader,
     _backendType = VectorDistanceUtils.resolveBackendType(vectorIndexConfig);
     _distanceFunction = VectorDistanceUtils.resolveDistanceFunction(vectorIndexConfig);
     _hasVectorIndex = vectorIndexReader != null;
+    _backendSupportsApproximateRadius = _backendType.getCapabilities().supportsApproximateRadius();
+    _readerSupportsApproximateRadius = vectorIndexReader instanceof ApproximateRadiusVectorIndexReader;
+    _useApproximateRadiusPath =
+        _hasVectorIndex && _backendSupportsApproximateRadius && _readerSupportsApproximateRadius;
   }
 
   @Override
@@ -134,11 +142,12 @@ public List<Operator> getChildOperators() {
 
   @Override
   public String toExplainString() {
-    return EXPLAIN_NAME + "(indexLookUp:" + (_hasVectorIndex ? "vector_index_with_scan" : "exact_scan")
+    return EXPLAIN_NAME + "(indexLookUp:" + getIndexLookupMode()
         + ", operator:" + _predicate.getType()
         + ", vector identifier:" + _column
         + ", backend:" + _backendType
         + ", distanceFunction:" + _distanceFunction
+        + ", approximateRadiusPath:" + _useApproximateRadiusPath
         + ", vector literal:" + Arrays.toString(_predicate.getValue())
         + ", threshold:" + _predicate.getThreshold()
         + ')';
@@ -152,11 +161,12 @@ protected String getExplainName() {
   @Override
   protected void explainAttributes(ExplainAttributeBuilder attributeBuilder) {
     super.explainAttributes(attributeBuilder);
-    attributeBuilder.putString("indexLookUp", _hasVectorIndex ? "vector_index_with_scan" : "exact_scan");
+    attributeBuilder.putString("indexLookUp", getIndexLookupMode());
     attributeBuilder.putString("operator", _predicate.getType().name());
     attributeBuilder.putString("vectorIdentifier", _column);
     attributeBuilder.putString("backend", _backendType.name());
     attributeBuilder.putString("distanceFunction", _distanceFunction.name());
+    attributeBuilder.putBool("approximateRadiusPath", _useApproximateRadiusPath);
     attributeBuilder.putString("vectorLiteral", Arrays.toString(_predicate.getValue()));
     attributeBuilder.putString("threshold", String.valueOf(_predicate.getThreshold()));
   }
@@ -183,11 +193,18 @@ private ImmutableRoaringBitmap executeSearch() {
    */
   private ImmutableRoaringBitmap executeIndexAssistedSearch(float[] queryVector, float threshold) {
     int internalLimit = Math.min(VectorSimilarityRadiusPredicate.DEFAULT_INTERNAL_LIMIT, _numDocs);
-    ImmutableRoaringBitmap candidates = _vectorIndexReader.getDocIds(queryVector, internalLimit);
+    ImmutableRoaringBitmap candidates;
+    if (_useApproximateRadiusPath) {
+      candidates =
+          ((ApproximateRadiusVectorIndexReader) _vectorIndexReader).getDocIdsWithinApproximateRadius(queryVector,
+              threshold, internalLimit);
+    } else {
+      candidates = _vectorIndexReader.getDocIds(queryVector, internalLimit);
+    }
 
     int candidateCount = candidates.getCardinality();
-    LOGGER.debug("Vector radius search on column: {}, candidates from index: {}, threshold: {}",
-        _column, candidateCount, threshold);
+    LOGGER.debug("Vector radius search on column: {}, candidates from index: {}, threshold: {}, source: {}",
+        _column, candidateCount, threshold, _useApproximateRadiusPath ? "approximate_radius_path" : "topk_path");
 
     if (candidateCount >= internalLimit) {
       // ANN candidate pool is saturated — fall back to exact brute-force scan to guarantee
@@ -202,6 +219,16 @@ private ImmutableRoaringBitmap executeIndexAssistedSearch(float[] queryVector, f
     return filterByThreshold(candidates, queryVector, threshold);
   }
 
+  private String getIndexLookupMode() {
+    if (!_hasVectorIndex) {
+      return "exact_scan";
+    }
+    if (_useApproximateRadiusPath) {
+      return "vector_index_approx_radius_with_scan";
+    }
+    return "vector_index_topk_with_scan";
+  }
+
   /**
    * Scans all documents and returns those within the distance threshold.
    */