Fix BPE non-determinism and add comprehensive test suite

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
       - name: Clippy
         run: cargo clippy --all-targets --workspace -- -D warnings
 
-  build-and-test:
+  rust-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -77,7 +77,7 @@ jobs:
         run: cargo build --all-targets
 
       - name: Test
-        run: cargo test
+        run: cargo test -- --skip test_read_java_written_file --skip test_read_python_written_file
 
   cpp-test:
     runs-on: ubuntu-latest
@@ -143,10 +143,16 @@ jobs:
       - name: Build JNI library
         run: cargo build -p paimon-mosaic-jni --release
 
+      - name: Write interop files (Rust)
+        run: cargo test --release -p paimon-mosaic-core --test interop_write_test
+
       - name: Run Java tests
         working-directory: java
         run: mvn test "-DargLine=-Djava.library.path=${{ github.workspace }}/target/release"
 
+      - name: Read Java-written interop file (Rust)
+        run: cargo test --release -p paimon-mosaic-core --test interop_read_test test_read_java_written_file
+
   python-test:
     runs-on: ubuntu-latest
     steps:
@@ -171,6 +177,9 @@ jobs:
       - name: Build FFI library
         run: cargo build --release -p paimon-mosaic-ffi
 
+      - name: Write interop files (Rust)
+        run: cargo test --release -p paimon-mosaic-core --test interop_write_test
+
       - name: Install Python dependencies
         working-directory: python
         run: pip install pyarrow pytest
@@ -180,3 +189,6 @@ jobs:
         run: pytest -v
         env:
           MOSAIC_LIB_PATH: ${{ github.workspace }}/target/release
+
+      - name: Read Python-written interop file (Rust)
+        run: cargo test --release -p paimon-mosaic-core --test interop_read_test test_read_python_written_file

core/src/bpe.rs

@@ -41,7 +41,7 @@ pub fn build_vocabulary(names: &[&[u8]]) -> Vec<[u8; 2]> {
             }
         }
 
-        let best = pair_counts.iter().max_by_key(|&(_, &v)| v);
+        let best = pair_counts.iter().max_by_key(|&(&k, &v)| (v, k));
         match best {
             Some((&pair, &count)) if count > 1 => {
                 let left = (pair >> 16) as u16;