[vector]support lumina

paimon-common/src/main/java/org/apache/paimon/utils/RoaringNavigableMap64.java

@@ -142,6 +142,22 @@ public List<Range> toRangeList() {
         return Range.toRanges(roaring64NavigableMap::iterator);
     }
 
+    /**
+     * Returns true if there is at least one value in the range [minId, maxId] (inclusive on both
+     * ends) contained in this bitmap.
+     *
+     * <p>Uses {@code rankLong} for O(log N) performance instead of iterating all values.
+     */
+    public boolean containsRange(long minId, long maxId) {
+        if (minId > maxId) {
+            throw new IllegalArgumentException(
+                    "minId (" + minId + ") must be <= maxId (" + maxId + ")");
+        }
+        long countUpToMax = roaring64NavigableMap.rankLong(maxId);
+        long countBeforeMin = minId > 0 ? roaring64NavigableMap.rankLong(minId - 1) : 0;
+        return countUpToMax > countBeforeMin;
+    }
+
     public static RoaringNavigableMap64 bitmapOf(long... dat) {
         RoaringNavigableMap64 roaringBitmap64 = new RoaringNavigableMap64();
         for (long ele : dat) {

paimon-lumina/README.md

@@ -0,0 +1,36 @@
+## Paimon Lumina
+
+This module integrates [Lumina](https://github.com/alibaba/paimon-cpp/tree/main/third_party/lumina)
+as a vector index for Apache Paimon's global index framework.
+
+Lumina vector search library is derived from an internal repository maintained by
+Alibaba Storage Service Team. It is accessed via JNI through the `lumina-jni` artifact.
+
+### Supported Index Types
+
+| Index Type | Description |
+|------------|-------------|
+| **DISKANN** | DiskANN graph-based index (default) |
+
+### Supported Vector Metrics
+
+| Metric | Description |
+|--------|-------------|
+| **L2** | Euclidean distance (default) |
+| **COSINE** | Cosine distance |
+| **INNER_PRODUCT** | Dot product |
+
+### Configuration Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `vector.dim` | int | 128 | Vector dimension |
+| `vector.metric` | enum | L2 | Distance metric |
+| `vector.index-type` | enum | DISKANN | Index type |
+| `vector.encoding-type` | string | rawf32 | Encoding type (rawf32, sq8, pq) |
+| `vector.size-per-index` | int | 2,000,000 | Max vectors per index file |
+| `vector.training-size` | int | 500,000 | Vectors used for pretraining |
+| `vector.search-factor` | int | 10 | Multiplier for search limit when filtering |
+| `vector.normalize` | boolean | false | L2-normalize vectors before indexing/searching |
+| `vector.diskann.search-list-size` | int | 100 | DiskANN search list size |
+| `vector.pretrain-sample-ratio` | double | 1.0 | Pretrain sample ratio |

paimon-lumina/pom.xml

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>paimon-parent</artifactId>
+        <groupId>org.apache.paimon</groupId>
+        <version>1.4-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>paimon-lumina</artifactId>
+    <name>Paimon : Lumina Index</name>
+
+    <repositories>
+        <repository>
+            <id>lumina</id>
+            <url>https://lumina-binary.oss-cn-shanghai.aliyuncs.com/mvn-repo/</url>
+        </repository>
+    </repositories>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-common</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.aliyun.lumina</groupId>
+            <artifactId>lumina-jni</artifactId>
+            <version>0.1.0</version>
+        </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${junit5.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-core</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-format</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-test-utils</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>${hadoop.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+</project>

paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaIndex.java

@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.lumina.index;
+
+import org.aliyun.lumina.LuminaBuilder;
+import org.aliyun.lumina.LuminaFileInput;
+import org.aliyun.lumina.LuminaFileOutput;
+import org.aliyun.lumina.LuminaSearcher;
+import org.aliyun.lumina.MetricType;
+
+import java.io.Closeable;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * A high-level wrapper for Lumina index operations (build and search).
+ *
+ * <p>This class provides a safe Java API for building and searching Lumina vector indices. It
+ * manages the lifecycle of native LuminaBuilder and LuminaSearcher objects.
+ */
+public class LuminaIndex implements Closeable {
+
+    private LuminaBuilder builder;
+    private LuminaSearcher searcher;
+    private final int dimension;
+    private final LuminaVectorMetric metric;
+    private final LuminaIndexType indexType;
+    private volatile boolean closed = false;
+
+    private LuminaIndex(int dimension, LuminaVectorMetric metric, LuminaIndexType indexType) {
+        this.dimension = dimension;
+        this.metric = metric;
+        this.indexType = indexType;
+    }
+
+    /** Create a new index for building. */
+    public static LuminaIndex createForBuild(
+            int dimension,
+            LuminaVectorMetric metric,
+            LuminaIndexType indexType,
+            Map<String, String> extraOptions) {
+        LuminaIndex index = new LuminaIndex(dimension, metric, indexType);
+
+        Map<String, String> opts = new LinkedHashMap<>(extraOptions);
+        index.builder =
+                LuminaBuilder.create(
+                        indexType.getLuminaName(), dimension, toMetricType(metric), opts);
+        return index;
+    }
+
+    /**
+     * Open an existing index from a streaming file input for searching.
+     *
+     * <p>The native searcher reads on-demand from the provided input. The caller must keep the
+     * underlying stream open until this index is closed.
+     */
+    public static LuminaIndex fromStream(
+            LuminaFileInput fileInput,
+            long fileSize,
+            int dimension,
+            LuminaVectorMetric metric,
+            LuminaIndexType indexType,
+            Map<String, String> extraOptions) {
+        LuminaIndex index = new LuminaIndex(dimension, metric, indexType);
+
+        Map<String, String> searcherOpts = new LinkedHashMap<>();
+        for (Map.Entry<String, String> entry : extraOptions.entrySet()) {
+            String key = entry.getKey();
+            if (key.startsWith("diskann.search.")) {
+                searcherOpts.put(key, entry.getValue());
+            }
+        }
+        index.searcher =
+                LuminaSearcher.create(
+                        indexType.getLuminaName(), dimension, toMetricType(metric), searcherOpts);
+        index.searcher.open(fileInput, fileSize);
+        return index;
+    }
+
+    /** Pretrain the index with sample vectors before insertion. */
+    public void pretrain(ByteBuffer vectorBuffer, int n) {
+        ensureOpen();
+        ensureBuilder();
+        builder.pretrain(vectorBuffer, n);
+    }
+
+    /** Insert a batch of vectors with IDs (zero-copy). */
+    public void insertBatch(ByteBuffer vectorBuffer, ByteBuffer idBuffer, int n) {
+        ensureOpen();
+        ensureBuilder();
+        builder.insertBatch(vectorBuffer, idBuffer, n);
+    }
+
+    /** Dump (serialize) the built index to a streaming file output. */
+    public void dump(LuminaFileOutput fileOutput) {
+        ensureOpen();
+        ensureBuilder();
+        builder.dump(fileOutput);
+    }
+
+    /** Search for k nearest neighbors. */
+    public void search(
+            float[] queryVectors,
+            int n,
+            int k,
+            float[] distances,
+            long[] labels,
+            Map<String, String> searchOptions) {
+        ensureOpen();
+        ensureSearcher();
+        searcher.search(n, queryVectors, k, distances, labels, searchOptions);
+    }
+
+    /** Search for k nearest neighbors with native pre-filtering on vector IDs. */
+    public void searchWithFilter(
+            float[] queryVectors,
+            int n,
+            int k,
+            float[] distances,
+            long[] labels,
+            long[] filterIds,
+            Map<String, String> searchOptions) {
+        ensureOpen();
+        ensureSearcher();
+        searcher.searchWithFilter(n, queryVectors, k, distances, labels, filterIds, searchOptions);
+    }
+
+    /** Get the number of vectors (searcher mode). */
+    public long size() {
+        ensureOpen();
+        ensureSearcher();
+        return searcher.getCount();
+    }
+
+    public int dimension() {
+        return dimension;
+    }
+
+    public LuminaVectorMetric metric() {
+        return metric;
+    }
+
+    public LuminaIndexType indexType() {
+        return indexType;
+    }
+
+    public static ByteBuffer allocateVectorBuffer(int numVectors, int dimension) {
+        return ByteBuffer.allocateDirect(numVectors * dimension * Float.BYTES)
+                .order(ByteOrder.nativeOrder());
+    }
+
+    public static ByteBuffer allocateIdBuffer(int numIds) {
+        return ByteBuffer.allocateDirect(numIds * Long.BYTES).order(ByteOrder.nativeOrder());
+    }
+
+    private void ensureOpen() {
+        if (closed) {
+            throw new IllegalStateException("Index has been closed");
+        }
+    }
+
+    private void ensureBuilder() {
+        if (builder == null) {
+            throw new IllegalStateException("Index was not opened for building");
+        }
+    }
+
+    private void ensureSearcher() {
+        if (searcher == null) {
+            throw new IllegalStateException("Index was not opened for searching");
+        }
+    }
+
+    private static MetricType toMetricType(LuminaVectorMetric metric) {
+        switch (metric) {
+            case L2:
+                return MetricType.L2;
+            case COSINE:
+                return MetricType.COSINE;
+            case INNER_PRODUCT:
+                return MetricType.INNER_PRODUCT;
+            default:
+                throw new IllegalArgumentException("Unknown metric: " + metric);
+        }
+    }
+
+    @Override
+    public void close() {
+        if (!closed) {
+            synchronized (this) {
+                if (!closed) {
+                    if (builder != null) {
+                        builder.close();
+                        builder = null;
+                    }
+                    if (searcher != null) {
+                        searcher.close();
+                        searcher = null;
+                    }
+                    closed = true;
+                }
+            }
+        }
+    }
+}