ci.yml

name: Build and Release

on:
  push:
    branches: [ main ]
  pull_request:

jobs:
  dotnet-build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: 9.0.x

      - name: Restore dependencies
        run: dotnet restore

      - name: Build managed projects
        run: dotnet build --configuration Release --no-restore

  native-assets:
    needs: dotnet-build
    runs-on: ubuntu-latest
    env:
      NATIVE_RELEASE_REPO: ${{ vars.MLXSHARP_NATIVE_REPO || 'ManagedCode/MLXSharp' }}
      NATIVE_RELEASE_TAG: ${{ vars.MLXSHARP_NATIVE_TAG || '' }}
    steps:
      - name: Install tooling
        run: |
          sudo apt-get update
          sudo apt-get install -y jq unzip

      - name: Download official native binaries
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -euo pipefail

          repo="${NATIVE_RELEASE_REPO}"
          if [ -z "$repo" ]; then
            echo "::error::NATIVE_RELEASE_REPO must be provided" >&2
            exit 1
          fi

          if [ -n "${NATIVE_RELEASE_TAG}" ]; then
            release_api="https://api.github.com/repos/${repo}/releases/tags/${NATIVE_RELEASE_TAG}"
          else
            release_api="https://api.github.com/repos/${repo}/releases/latest"
          fi

          echo "Fetching release metadata from ${release_api}"
          response=$(curl -fsSL -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" "$release_api")
          tag=$(echo "$response" | jq -r '.tag_name // ""')

          if [ -z "$tag" ]; then
            echo "::error::Unable to resolve release tag from ${release_api}" >&2
            exit 1
          fi

          echo "Using native release ${repo}@${tag}"

          nupkg_asset=$(echo "$response" | jq -r '.assets[] | select(.name | startswith("ManagedCode.MLXSharp.")) | select(.name | endswith(".nupkg")) | .name' | head -n1)
          if [ -z "$nupkg_asset" ] || [ "$nupkg_asset" = "null" ]; then
            echo "::error::No ManagedCode.MLXSharp.*.nupkg asset found in release ${tag}" >&2
            exit 1
          fi

          asset_url=$(echo "$response" | jq -r --arg name "$nupkg_asset" '.assets[] | select(.name == $name) | .url')
          if [ -z "$asset_url" ] || [ "$asset_url" = "null" ]; then
            echo "::error::Failed to resolve download URL for ${nupkg_asset}" >&2
            exit 1
          fi

          mkdir -p work artifacts/native/osx-arm64 artifacts/native/linux-x64

          echo "Downloading ${nupkg_asset}"
          curl -fsSL -H "Accept: application/octet-stream" -H "Authorization: Bearer ${GITHUB_TOKEN}" "$asset_url" -o work/native.nupkg

          echo "Extracting native runtimes"
          unzip -q work/native.nupkg 'runtimes/osx-arm64/native/*' -d work/extract
          unzip -q work/native.nupkg 'runtimes/linux-x64/native/*' -d work/extract

          shopt -s nullglob
          mac_files=(work/extract/runtimes/osx-arm64/native/*)
          linux_files=(work/extract/runtimes/linux-x64/native/*)

          if [ ${#mac_files[@]} -eq 0 ]; then
            echo "::error::macOS native assets are missing from ${nupkg_asset}" >&2
            exit 1
          fi

          if [ ${#linux_files[@]} -eq 0 ]; then
            echo "::error::Linux native assets are missing from ${nupkg_asset}" >&2
            exit 1
          fi

          cp work/extract/runtimes/osx-arm64/native/* artifacts/native/osx-arm64/
          cp work/extract/runtimes/linux-x64/native/* artifacts/native/linux-x64/

          echo "Staged native artifacts:"
          ls -R artifacts/native

      - name: Upload macOS native artifact
        uses: actions/upload-artifact@v4
        with:
          name: native-osx-arm64
          path: artifacts/native/osx-arm64

      - name: Upload Linux native artifact
        uses: actions/upload-artifact@v4
        with:
          name: native-linux-x64
          path: artifacts/native/linux-x64

  package-test:
    needs:
      - native-assets
    runs-on: macos-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Setup .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: 9.0.x

      - name: Show .NET info
        run: dotnet --info

      - name: Restore dependencies
        run: dotnet restore 

      - name: Build C# projects (initial validation)
        run: dotnet build  --configuration Release --no-restore

      - name: Setup Python for HuggingFace
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install huggingface_hub mlx mlx-lm

      - name: Download test model from HuggingFace
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        run: |
          mkdir -p models
          python - <<'PY'
          import os
          from pathlib import Path

          from huggingface_hub import snapshot_download

          target_dir = Path("models/Qwen1.5-0.5B-Chat-4bit")
          target_dir.mkdir(parents=True, exist_ok=True)

          snapshot_download(
              repo_id="mlx-community/Qwen1.5-0.5B-Chat-4bit",
              local_dir=str(target_dir),
              local_dir_use_symlinks=False,
              token=os.environ.get("HF_TOKEN") or None,
              resume_download=True,
          )
          PY
          echo "Model files:"
          ls -la models/Qwen1.5-0.5B-Chat-4bit/

      - name: Download macOS native library
        uses: actions/download-artifact@v4
        with:
          name: native-osx-arm64
          path: artifacts/native/osx-arm64

      - name: Download Linux native library
        uses: actions/download-artifact@v4
        with:
          name: native-linux-x64
          path: artifacts/native/linux-x64

      - name: Ensure macOS metallib is available
        run: |
          set -euo pipefail

          metallib_path="artifacts/native/osx-arm64/mlx.metallib"
          if [ -f "${metallib_path}" ]; then
            echo "Found mlx.metallib in downloaded native artifact."
            exit 0
          fi

          echo "::warning::mlx.metallib missing from native artifact; attempting to source from installed mlx package"
          python - <<'PY'
          import importlib.util
          from importlib import resources
          import pathlib
          import shutil
          import sys
          from typing import Iterable, Optional

          try:
              import mlx  # type: ignore
          except ImportError:
              print("::error::The 'mlx' Python package is not installed; cannot locate mlx.metallib.")
              sys.exit(1)

          search_dirs: list[pathlib.Path] = []
          package_dir: Optional[pathlib.Path] = None
          package_paths: list[pathlib.Path] = []

          package_file = getattr(mlx, "__file__", None)
          if package_file:
              try:
                  package_paths.append(pathlib.Path(package_file).resolve().parent)
              except (TypeError, OSError):
                  pass

          package_path_attr = getattr(mlx, "__path__", None)
          if package_path_attr:
              for entry in package_path_attr:
                  try:
                      package_paths.append(pathlib.Path(entry).resolve())
                  except (TypeError, OSError):
                      continue

          try:
              spec = importlib.util.find_spec("mlx.backend.metal.kernels")
          except ModuleNotFoundError:
              spec = None

          if spec and spec.origin:
              candidate = pathlib.Path(spec.origin).resolve().parent
              if candidate.exists():
                  search_dirs.append(candidate)
                  package_paths.append(candidate)

          def append_resource_directory(module: str, *subpath: str) -> None:
              try:
                  traversable = resources.files(module)
              except (ModuleNotFoundError, AttributeError):
                  return

              for segment in subpath:
                  traversable = traversable / segment

              try:
                  with resources.as_file(traversable) as extracted:
                      if extracted:
                          extracted_path = pathlib.Path(extracted).resolve()
                          if extracted_path.exists():
                              search_dirs.append(extracted_path)
                              package_paths.append(extracted_path)
              except (FileNotFoundError, RuntimeError):
                  pass

          append_resource_directory("mlx.backend.metal", "kernels")
          append_resource_directory("mlx")

          existing_package_paths: list[pathlib.Path] = []
          seen_package_paths: set[pathlib.Path] = set()
          for path in package_paths:
              if not path:
                  continue
              try:
                  resolved = path.resolve()
              except (OSError, RuntimeError):
                  continue
              if not resolved.exists():
                  continue
              if resolved in seen_package_paths:
                  continue
              seen_package_paths.add(resolved)
              existing_package_paths.append(resolved)

          if existing_package_paths:
              package_dir = existing_package_paths[0]
              for root in existing_package_paths:
                  search_dirs.extend(
                      [
                          root / "backend" / "metal" / "kernels",
                          root / "backend" / "metal",
                          root,
                      ]
                  )

          ordered_dirs: list[pathlib.Path] = []
          seen: set[pathlib.Path] = set()
          for candidate in search_dirs:
              if not candidate:
                  continue
              candidate = candidate.resolve()
              if candidate in seen:
                  continue
              seen.add(candidate)
              ordered_dirs.append(candidate)

          def iter_metallibs(dirs: Iterable[pathlib.Path]):
              for directory in dirs:
                  if not directory.exists():
                      continue
                  preferred = directory / "mlx.metallib"
                  if preferred.exists():
                      yield preferred
                      continue
                  for alternative in sorted(directory.glob("*.metallib")):
                      yield alternative

          src = next(iter_metallibs(ordered_dirs), None)

          package_roots = existing_package_paths if existing_package_paths else ([] if not package_dir else [package_dir])

          if src is None:
              for root in package_roots:
                  for candidate in root.rglob("mlx.metallib"):
                      src = candidate
                      print(f"::warning::Resolved metallib via recursive search under {root}")
                      break
                  if src is not None:
                      break

          if src is None:
              for root in package_roots:
                  for candidate in sorted(root.rglob("*.metallib")):
                      src = candidate
                      print(f"::warning::Using metallib {candidate.name} discovered via package-wide search in {root}")
                      break
                  if src is not None:
                      break

          if src is None:
              print("::error::Could not locate any mlx.metallib artifacts within the installed mlx package.")
              sys.exit(1)

          if src.name != "mlx.metallib":
              print(f"::warning::Using metallib {src.name} from {src.parent}")

          dest = pathlib.Path("artifacts/native/osx-arm64/mlx.metallib").resolve()
          dest.parent.mkdir(parents=True, exist_ok=True)

          shutil.copy2(src, dest)
          print(f"Copied mlx.metallib from {src} to {dest}")
          PY

      - name: Stage native libraries in project
        run: |
          set -euo pipefail

          mkdir -p src/MLXSharp/runtimes/osx-arm64/native
          cp artifacts/native/osx-arm64/libmlxsharp.dylib src/MLXSharp/runtimes/osx-arm64/native/

          if [ -f artifacts/native/osx-arm64/mlx.metallib ]; then
            cp artifacts/native/osx-arm64/mlx.metallib src/MLXSharp/runtimes/osx-arm64/native/
          else
            echo "::warning::mlx.metallib not found in macOS native artifact; continuing without Metal shaders"
          fi

          mkdir -p src/MLXSharp/runtimes/linux-x64/native
          cp artifacts/native/linux-x64/libmlxsharp.so src/MLXSharp/runtimes/linux-x64/native/

      - name: Build C# projects with native libraries
        run: dotnet build --configuration Release --no-restore

      - name: Copy native library to test output
        run: |
          TEST_OUTPUT="src/MLXSharp.Tests/bin/Release/net9.0"
          mkdir -p "$TEST_OUTPUT/runtimes/osx-arm64/native"
          cp src/MLXSharp/runtimes/osx-arm64/native/libmlxsharp.dylib "$TEST_OUTPUT/runtimes/osx-arm64/native/"

          if [ -f src/MLXSharp/runtimes/osx-arm64/native/mlx.metallib ]; then
            cp src/MLXSharp/runtimes/osx-arm64/native/mlx.metallib "$TEST_OUTPUT/runtimes/osx-arm64/native/"
          else
            echo "::warning::mlx.metallib not staged; tests will continue without Metal shaders"
          fi
          ls -la "$TEST_OUTPUT/runtimes/osx-arm64/native/"

      - name: Run tests
        run: |
          dotnet test \
            --configuration Release \
            --no-build \
            --logger "trx;LogFileName=TestResults.trx" \
            --logger "console;verbosity=detailed" \
            --results-directory artifacts/test-results
        env:
          MLXSHARP_MODEL_PATH: ${{ github.workspace }}/models/Qwen1.5-0.5B-Chat-4bit

      - name: Prepare artifact folders
        run: |
          mkdir -p artifacts/test-results
          mkdir -p artifacts/packages
          mkdir -p artifacts/native
          cp artifacts/native/osx-arm64/libmlxsharp.dylib artifacts/native/
          cp artifacts/native/osx-arm64/mlx.metallib artifacts/native/
          cp artifacts/native/linux-x64/libmlxsharp.so artifacts/native/

      - name: Pack MLXSharp library
        run: dotnet pack src/MLXSharp/MLXSharp.csproj --configuration Release --output artifacts/packages -p:MLXSharpMacNativeBinary=$GITHUB_WORKSPACE/artifacts/native/osx-arm64/libmlxsharp.dylib -p:MLXSharpMacMetallibBinary=$GITHUB_WORKSPACE/artifacts/native/osx-arm64/mlx.metallib -p:MLXSharpLinuxNativeBinary=$GITHUB_WORKSPACE/artifacts/native/linux-x64/libmlxsharp.so

      - name: Pack MLXSharp.SemanticKernel library
        run: dotnet pack src/MLXSharp.SemanticKernel/MLXSharp.SemanticKernel.csproj --configuration Release --output artifacts/packages -p:MLXSharpMacNativeBinary=$GITHUB_WORKSPACE/artifacts/native/osx-arm64/libmlxsharp.dylib -p:MLXSharpMacMetallibBinary=$GITHUB_WORKSPACE/artifacts/native/osx-arm64/mlx.metallib -p:MLXSharpLinuxNativeBinary=$GITHUB_WORKSPACE/artifacts/native/linux-x64/libmlxsharp.so -p:MLXSharpSkipLinuxNativeValidation=true

      - name: Verify package contains native libraries
        run: |
          echo "Checking package contents..."
          shopt -s nullglob
          packages=(artifacts/packages/*.nupkg)
          if [ ${#packages[@]} -eq 0 ]; then
            echo "✗ ERROR: No packages were produced"
            exit 1
          fi

          missing=0
          for package in "${packages[@]}"; do
            echo "Inspecting ${package}"
            filename=$(basename "${package}")
            case "${filename}" in
              MLXSharp.SemanticKernel.*.nupkg)
                echo "  ↷ Skipping native check for ${filename}"
                ;;
              MLXSharp.*.nupkg)
                package_missing=0
                if unzip -l "${package}" | grep -q "runtimes/osx-arm64/native/libmlxsharp.dylib"; then
                  echo "  ✓ macOS library present"
                else
                  echo "  ✗ macOS library missing"
                  package_missing=1
                fi
                if unzip -l "${package}" | grep -q "runtimes/osx-arm64/native/mlx.metallib"; then
                  echo "  ✓ macOS metallib present"
                else
                  echo "  ✗ macOS metallib missing"
                  package_missing=1
                fi
                if unzip -l "${package}" | grep -q "runtimes/linux-x64/native/libmlxsharp.so"; then
                  echo "  ✓ Linux library present"
                else
                  echo "  ✗ Linux library missing"
                  package_missing=1
                fi

                if [ ${package_missing} -ne 0 ]; then
                  unzip -l "${package}"
                  missing=1
                fi
                ;;
              *)
                echo "  ↷ Skipping native check for ${filename}"
                ;;
            esac
          done

          if [ $missing -ne 0 ]; then
            exit 1
          fi

      - name: Upload native artifact
        uses: actions/upload-artifact@v4
        with:
          name: native-libs
          path: artifacts/native

      - name: Upload packages artifact
        uses: actions/upload-artifact@v4
        with:
          name: packages
          path: artifacts/packages

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: test-results
          path: artifacts/test-results

      - name: Publish test results summary
        if: always()
        run: |
          python - <<'PY'
          import os
          import xml.etree.ElementTree as ET

          trx_path = "artifacts/test-results/TestResults.trx"
          summary_path = os.environ.get("GITHUB_STEP_SUMMARY")

          if not os.path.exists(trx_path) or not summary_path:
              print("No test results found to summarize.")
              raise SystemExit(0)

          ns = {"trx": "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"}
          root = ET.parse(trx_path).getroot()
          counters = root.find(".//trx:ResultSummary/trx:Counters", ns)

          with open(summary_path, "a", encoding="utf-8") as summary:
              summary.write("### Test Results\n\n")

              if counters is not None:
                  metrics = {
                      "Total": counters.attrib.get("total", "0"),
                      "Executed": counters.attrib.get("executed", "0"),
                      "Passed": counters.attrib.get("passed", "0"),
                      "Failed": counters.attrib.get("failed", "0"),
                      "Errors": counters.attrib.get("error", "0"),
                      "Timeouts": counters.attrib.get("timeout", "0"),
                      "Aborted": counters.attrib.get("aborted", "0"),
                      "Inconclusive": counters.attrib.get("inconclusive", "0"),
                      "Skipped": counters.attrib.get("notExecuted", "0")
                  }

                  for label, value in metrics.items():
                      if value and value != "0":
                          summary.write(f"- {label}: {value}\n")
              else:
                  summary.write("- No counters were present in the TRX log.\n")

              failed_results = root.findall(".//trx:UnitTestResult[@outcome!='Passed']", ns)
              if failed_results:
                  summary.write("\nFailed Tests:\n")
                  for result in failed_results:
                      test_name = result.attrib.get("testName", "(unknown test)")
                      outcome = result.attrib.get("outcome", "Unknown")
                      summary.write(f"- {test_name} ({outcome})\n")

              summary.write("\n")
          PY

  release:
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    needs: package-test
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive

#      - name: Determine MLX version
#        id: mlx_version
#        run: |
#          git submodule update --init --recursive
#          git -C extern/mlx fetch --tags
#
#          if MLX_TAG=$(git -C extern/mlx describe --tags --abbrev=0 2>/dev/null); then
#            echo "Detected MLX tag: ${MLX_TAG}"
#          else
#            echo "::warning::Unable to determine MLX tag; falling back to commit hash"
#            MLX_TAG="unknown"
#          fi
#
#          MLX_COMMIT=$(git -C extern/mlx rev-parse --short HEAD)
#
#          echo "tag=${MLX_TAG}" >> "$GITHUB_OUTPUT"
#          echo "commit=${MLX_COMMIT}" >> "$GITHUB_OUTPUT"
#
#      - name: Extract version from Directory.Build.props
#        id: version
#        run: |
#          VERSION=$(grep -oP '<Version>\K[^<]+' Directory.Build.props)
#          echo "version=$VERSION" >> $GITHUB_OUTPUT
#          echo "Extracted version: $VERSION"
#
#      - name: Check if tag exists
#        id: check_tag
#        run: |
#          if git ls-remote --tags origin | grep -q "refs/tags/v${{ steps.version.outputs.version }}"; then
#            echo "exists=true" >> $GITHUB_OUTPUT
#            echo "Tag v${{ steps.version.outputs.version }} already exists"
#          else
#            echo "exists=false" >> $GITHUB_OUTPUT
#            echo "Tag v${{ steps.version.outputs.version }} does not exist"
#          fi
#
#      - name: Download package artifacts
#        if: steps.check_tag.outputs.exists == 'false'
#        uses: actions/download-artifact@v4
#        with:
#          name: packages
#          path: artifacts/packages
#
#      - name: Publish to NuGet
#        if: steps.check_tag.outputs.exists == 'false'
#        run: |
#          for package in artifacts/packages/*.nupkg; do
#            echo "Publishing $package..."
#            dotnet nuget push "$package" \
#              --api-key ${{ secrets.NUGET_API_KEY }} \
#              --source https://api.nuget.org/v3/index.json \
#              --skip-duplicate || true
#          done
#
#      - name: Create Git tag
#        if: steps.check_tag.outputs.exists == 'false'
#        run: |
#          git config user.name "github-actions[bot]"
#          git config user.email "github-actions[bot]@users.noreply.github.com"
#          git tag -a "v${{ steps.version.outputs.version }}" -m "Release v${{ steps.version.outputs.version }}"
#          git push origin "v${{ steps.version.outputs.version }}"
#
#      - name: Generate release notes
#        if: steps.check_tag.outputs.exists == 'false'
#        id: release_notes
#        env:
#          RELEASE_VERSION: ${{ steps.version.outputs.version }}
#          MLX_TAG: ${{ steps.mlx_version.outputs.tag }}
#          MLX_COMMIT: ${{ steps.mlx_version.outputs.commit }}
#        run: |
#          PREVIOUS_TAG=$(git describe --abbrev=0 --tags $(git rev-list --tags --skip=1 --max-count=1) 2>/dev/null || echo "")
#          if [ -z "$PREVIOUS_TAG" ]; then
#            COMMITS=$(git log --pretty=format:"- %s (%h)" --reverse)
#          else
#            COMMITS=$(git log ${PREVIOUS_TAG}..HEAD --pretty=format:"- %s (%h)" --reverse)
#          fi
#
#          echo "## What's Changed" > release_notes.md
#          echo "" >> release_notes.md
#          echo "$COMMITS" >> release_notes.md
#          echo "" >> release_notes.md
#          echo "## Upstream MLX" >> release_notes.md
#          echo "- Version: ${MLX_TAG}" >> release_notes.md
#          echo "- Commit: ${MLX_COMMIT}" >> release_notes.md
#          echo "" >> release_notes.md
#          echo "## NuGet Packages" >> release_notes.md
#          echo "- [MLXSharp v${RELEASE_VERSION}](https://www.nuget.org/packages/MLXSharp/${RELEASE_VERSION})" >> release_notes.md
#          echo "- [MLXSharp.SemanticKernel v${RELEASE_VERSION}](https://www.nuget.org/packages/MLXSharp.SemanticKernel/${RELEASE_VERSION})" >> release_notes.md
#
#          cat release_notes.md
#
#      - name: Create GitHub Release
#        if: steps.check_tag.outputs.exists == 'false'
#        uses: softprops/action-gh-release@v1
#        with:
#          tag_name: v${{ steps.version.outputs.version }}
#          name: Release v${{ steps.version.outputs.version }}
#          body_path: release_notes.md
#          files: artifacts/packages/*
#        env:
#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}