Boost the performance of the inference

.github/workflows/build_template.yml

@@ -94,7 +94,7 @@ jobs:
         RPM_PACKAGE_VERSION: ${{ inputs.rpm_package_version }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           set-safe-directory: true
@@ -144,13 +144,17 @@ jobs:
           path: cache
           enableCrossOsArchive: true
           key: ${{ inputs.cache_key }}_${{ steps.branch.outputs.branch_name }}
+          restore-keys: |
+            ${{ inputs.cache_key }}_master
 
       - name: Check out deps cache before building
         uses: actions/cache@v4
         with:
           path: build/_deps/cache
           enableCrossOsArchive: true
           key: ${{ inputs.cache_key }}_deps_${{ steps.branch.outputs.branch_name }}
+          restore-keys: |
+            ${{ inputs.cache_key }}_deps_master
 
       - name: 🚀 Build
         run: ${{ inputs.cmake_command }}

.github/workflows/embedding_build_template.yml

@@ -23,7 +23,7 @@ jobs:
         shell: bash
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Rust toolchain
         uses: actions-rs/toolchain@v1
@@ -36,16 +36,29 @@ jobs:
         uses: arduino/setup-protoc@v3
 
       - name: Cache cargo registry
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
-            ./embeddings/target
-            ./embeddings/.cache/manticore
-          key: quality-check-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          key: cargo-registry-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}
+          restore-keys: cargo-registry-${{ runner.os }}-
+
+      - name: Cache cargo build target
+        uses: actions/cache@v4
+        with:
+          path: ./embeddings/target
+          key: cargo-target-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}-${{ hashFiles('embeddings/src/**/*.rs') }}
           restore-keys: |
-            quality-check-${{ runner.os }}-cargo-
+            cargo-target-${{ runner.os }}-${{ hashFiles('embeddings/Cargo.lock') }}-
+            cargo-target-${{ runner.os }}-
+
+      - name: Cache ML test models
+        uses: actions/cache@v4
+        with:
+          path: ./embeddings/.cache/manticore
+          key: ml-models-${{ runner.os }}-${{ hashFiles('embeddings/src/model/local_test.rs') }}
+          restore-keys: ml-models-${{ runner.os }}-
 
       - name: Run Clippy (strict mode)
         run: cargo clippy --lib --manifest-path ./embeddings/Cargo.toml -- -D warnings
@@ -87,6 +100,13 @@ jobs:
           JINA_API_KEY: ${{ secrets.JINA_API_KEY }}
           RUST_BACKTRACE: 1
 
+      - name: Clean build target bloat
+        if: always()
+        run: |
+          rm -rf ./embeddings/target/debug/incremental
+          rm -rf ./embeddings/target/debug/.fingerprint
+          find ./embeddings/target/debug -name "*.d" -delete 2>/dev/null || true
+
       - name: Quality check summary
         run: |
           echo "✅ Quality Check Summary:"
@@ -139,7 +159,10 @@ jobs:
           lib_dir="./embeddings/target/${target}/release"
           echo "lib_dir=${lib_dir}" >> $GITHUB_OUTPUT
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           target: ${{ steps.vars.outputs.target }}
@@ -168,41 +191,81 @@ jobs:
       - name: Prepare build args
         id: params
         run: |
+          # Enable platform-specific BLAS acceleration:
+          # - macOS: Apple Accelerate framework (always available)
+          # - Linux/Windows x86_64: Intel MKL (downloaded at build time)
+          # - Linux aarch64: no extra BLAS (gemm crate uses NEON via runtime detection)
+          features=""
+          if [[ "${{ inputs.distr }}" == "macos" ]]; then
+            features="--features accelerate"
+          elif [[ "${{ inputs.arch }}" == "x86_64" ]]; then
+            features="--features mkl"
+          fi
+
           args=(
-            "--target" 
-            "${{ steps.vars.outputs.target }}" 
+            "--target"
+            "${{ steps.vars.outputs.target }}"
             "--lib"
-            "--release" 
-            "--manifest-path" 
+            "--release"
+            "--manifest-path"
             "./embeddings/Cargo.toml"
+            ${features}
           )
           echo "args=${args[@]}" >> $GITHUB_OUTPUT
 
+      - name: Cache cargo registry for Docker build
+        if: ${{ inputs.distr == 'linux' }}
+        uses: actions/cache@v4
+        with:
+          path: |
+            .cargo-cache/registry
+            .cargo-cache/git
+          key: emb-build-cargo-${{ runner.os }}-${{ inputs.arch }}-${{ hashFiles('embeddings/Cargo.lock') }}
+          restore-keys: emb-build-cargo-${{ runner.os }}-${{ inputs.arch }}-
+
       - name: Build for Linux
         if: ${{ inputs.distr == 'linux' }}
         run: |
           # Set Docker image based on architecture
           if [[ "${{ inputs.arch }}" == "aarch64" ]]; then
-            docker_image="ghcr.io/manticoresoftware/rust-min-libc:aarch64-rust1.92.0-glibc2.27-openssl1.1.1k"
+            docker_image="ghcr.io/manticoresoftware/rust-min-libc:aarch64-rust1.94.1-glibc2.27-openssl1.1.1k"
             extra=""
           else
-            docker_image="ghcr.io/manticoresoftware/rust-min-libc:amd64-rust1.92.0-glibc2.27-openssl1.1.1k"
+            docker_image="ghcr.io/manticoresoftware/rust-min-libc:amd64-rust1.94.1-glibc2.27-openssl1.1.1k"
             extra=""
           fi
 
           docker run --rm  \
             -v $(pwd):/src \
+            -v $(pwd)/.cargo-cache:/cargo-cache \
             -w /src \
             -u root \
+            -e CARGO_HOME=/cargo-cache \
+            -e MKLROOT=/opt/intel/oneapi/mkl/latest \
             -e GIT_COMMIT_ID="${{ steps.git_meta.outputs.commit }}" \
             -e GIT_TIMESTAMP_ID="${{ steps.git_meta.outputs.timestamp }}" \
             --entrypoint bash \
             "${docker_image}" \
             -c "apt-get update -y && apt-get install -y gcc g++ protobuf-compiler && ${extra} cargo build ${{ steps.params.outputs.args }}"
           sudo chown -hR $(id -u):$(id -g) ${{ steps.vars.outputs.lib_dir }}
+          sudo chown -hR $(id -u):$(id -g) .cargo-cache
 
 
-      - name: Build native 
+      - name: Download ONNX Runtime static library
+        if: ${{ inputs.distr == 'macos' }}
+        run: |
+          ort_version="1.24.2"
+          if [[ "${{ inputs.arch }}" == "aarch64" ]]; then
+            ort_arch="arm64"
+          else
+            ort_arch="x86_64"
+          fi
+          ort_name="onnxruntime-osx-${ort_arch}-static_lib-${ort_version}"
+          curl -sL "https://github.com/csukuangfj/onnxruntime-libs/releases/download/v${ort_version}/${ort_name}.zip" -o /tmp/ort.zip
+          unzip -q /tmp/ort.zip -d /tmp/ort
+          echo "ORT_LIB_LOCATION=/tmp/ort/${ort_name}/lib" >> $GITHUB_ENV
+
+      - name: Build native
         if: ${{ inputs.distr == 'windows' || inputs.distr == 'macos' }}
         run: |
           rustup target add ${{ steps.vars.outputs.target }}

.github/workflows/mirror.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0

.github/workflows/nightly_no_avx2.yml

@@ -148,7 +148,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test report artifact 1_54

.github/workflows/pack_publish.yml

@@ -39,7 +39,7 @@ jobs:
       || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.ref == 'refs/heads/master' && github.event.workflow_run.event != 'schedule')
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -95,41 +95,6 @@ jobs:
             echo "Set manticore_locator: (empty)"
           fi
 
-  win_bundle:
-    name: Windows supplementary files preparation
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Determine branch name for cache
-        id: branch
-        run: |
-          if [ "${{ github.event_name }}" = "pull_request" ]; then
-            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_OUTPUT
-          else
-            echo "branch_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT
-          fi
-      - name: Check out cache
-        id: cache
-        uses: actions/cache@v4
-        with:
-          path: |
-            bundle
-            boost_1_75_0
-          enableCrossOsArchive: true
-          key: win_bundle_${{ steps.branch.outputs.branch_name }}
-          lookup-only: true
-      - name: Extract Windows bundle from Windows sysroot
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          wget https://repo.manticoresearch.com/repository/sysroots/roots_nov22/sysroot_windows_x64.tar.xz
-          tar -xvf sysroot_windows_x64.tar.xz
-          mv diskc/winbundle bundle
-      - name: Extract Boost to put it to the cache
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: |
-          wget https://repo.manticoresearch.com/repository/ci/boost_1_75_0.tgz
-          tar -xf boost_1_75_0.tgz
-
-
   embedding_linux_pack:
     needs: commit_info
     uses: ./.github/workflows/embedding_build_template.yml
@@ -236,7 +201,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test report artifact 1_54
@@ -362,7 +327,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test report artifact 1_54
@@ -441,7 +406,7 @@ jobs:
 
   test_windows:
     name: Windows tests
-    needs: [build_windows, win_bundle, check_branch]
+    needs: [build_windows, check_branch]
     uses: ./.github/workflows/win_test_template.yml
     strategy:
       fail-fast: false
@@ -478,7 +443,7 @@ jobs:
       image: manticoresearch/ubertests_ctest:3263_mar_2024
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test report artifact 1_86
@@ -570,13 +535,13 @@ jobs:
       image: ${{ steps.vars.outputs.image }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           fetch-tags: true
 
       - name: Checkout daemon repository for branch detection
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: manticoresoftware/manticoresearch
           path: manticoresearch
@@ -678,6 +643,17 @@ jobs:
             'apt-get remove -y manticore-columnar-lib || true && \
              dpkg -i --force-confnew /build/manticore-columnar-lib*.deb && \
              apt-get install -f -y'
+
+          # Debug: check installed libraries and their dependencies
+          docker exec temp bash -c \
+            'echo "=== Installed MCL files ===" && \
+             dpkg -L manticore-columnar-lib && \
+             echo "=== Embeddings lib dependencies ===" && \
+             find / -name "lib_manticore_knn_embeddings*" 2>/dev/null && \
+             for f in $(find / -name "lib_manticore_knn_embeddings*" 2>/dev/null); do ldd "$f" 2>&1; done && \
+             echo "=== KNN lib dependencies ===" && \
+             find / -name "lib_manticore_knn.*" 2>/dev/null && \
+             for f in $(find / -name "lib_manticore_knn.*" 2>/dev/null); do ldd "$f" 2>&1; done'
 
           # Commit the modified test-kit images
           docker commit temp ${{ steps.vars.outputs.image }}

.github/workflows/test_template.yml

@@ -60,7 +60,7 @@ jobs:
         # CTEST_REGEX: test_234
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -119,13 +119,15 @@ jobs:
           path: cache
           enableCrossOsArchive: true
           key: build_linux_x86_64_${{ steps.branch.outputs.branch_name }}
+          restore-keys: build_linux_x86_64_master
 
       - name: Check out deps cache before building
         uses: actions/cache@v4
         with:
           path: build/_deps/cache
           enableCrossOsArchive: true
           key: build_linux_x86_64_deps_${{ steps.branch.outputs.branch_name }}
+          restore-keys: build_linux_x86_64_deps_master
 
       - name: List files
         run: find .