dev-publish.yml

name: Dev Build & Publish

on:
  schedule:
    - cron: '0 * * * *'   # Every hour
  workflow_dispatch:

concurrency:
  group: dev-build-publish
  cancel-in-progress: false

permissions:
  contents: read
  actions: read

env:
  GCP_REGION: ${{ vars.GCP_REGION }}
  GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
  GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }}
  PROTOC_VERSION: "28.3"
  # arduino/setup-protoc@v3.0.0 has no Node.js 24 release yet; remove once a node24 version ships
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  # ---------------------------------------------------------------------------
  # Gate: skip scheduled runs when nothing changed since last success
  # ---------------------------------------------------------------------------
  check-changes:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          fetch-depth: 0

      - name: Check for changes since last successful run
        id: check
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
            echo "Manual trigger — always run"
            exit 0
          fi

          last_success=$(gh run list \
            --workflow "Dev Build & Publish" \
            --status success \
            --json updatedAt \
            --jq '.[0].updatedAt // empty')

          if [ -z "$last_success" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
            echo "No prior successful run found — running"
            exit 0
          fi

          commit_count=$(git rev-list --count --since="$last_success" HEAD)
          if [ "$commit_count" -gt 0 ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
            echo "$commit_count new commit(s) since $last_success"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
            echo "No new commits since $last_success — skipping"
          fi

  # ---------------------------------------------------------------------------
  # Stage 1: Compute dev version
  # ---------------------------------------------------------------------------
  version:
    needs: check-changes
    if: needs.check-changes.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      dev_version: ${{ steps.compute.outputs.dev_version }}
      cargo_version: ${{ steps.compute.outputs.cargo_version }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Compute dev version
        id: compute
        run: |
          base_version=$(python3 -c "
          import tomllib
          with open('pyproject.toml', 'rb') as f:
              print(tomllib.load(f)['project']['version'])
          ")
          timestamp=$(date +%s)
          short_sha=$(git rev-parse --short HEAD)
          dev_version="${base_version}.dev${timestamp}+g${short_sha}"
          cargo_version="${base_version}-dev.${timestamp}+g${short_sha}"
          echo "dev_version=${dev_version}" >> "$GITHUB_OUTPUT"
          echo "cargo_version=${cargo_version}" >> "$GITHUB_OUTPUT"
          echo "PEP 440 version (pyproject.toml): ${dev_version}"
          echo "SemVer version  (Cargo.toml):     ${cargo_version}"
          echo "${dev_version}" > dev_version.txt
          echo "${cargo_version}" > cargo_version.txt

      - name: Upload version artifact
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: dev-version
          path: |
            dev_version.txt
            cargo_version.txt

  # ---------------------------------------------------------------------------
  # Stage 2: Build wheels (matrix) + sdist
  # ---------------------------------------------------------------------------
  build-wheels:
    needs: version
    strategy:
      fail-fast: true
      matrix:
        include:
          # Linux glibc x86_64
          - os: ubuntu-latest
            target: x86_64-unknown-linux-gnu
            manylinux: auto
          # Linux glibc aarch64 (native ARM64 runner — avoids ring crate cross-compilation issues)
          - os: ubuntu-24.04-arm
            target: aarch64-unknown-linux-gnu
            manylinux: auto
          # Linux musl x86_64 (Alpine Docker images)
          - os: ubuntu-latest
            target: x86_64-unknown-linux-musl
            manylinux: musllinux_1_2
          # Linux musl aarch64 (Alpine on ARM)
          - os: ubuntu-24.04-arm
            target: aarch64-unknown-linux-musl
            manylinux: musllinux_1_2
          # macOS x86_64 (cross-compiled on arm64 runner)
          - os: macos-14
            target: x86_64-apple-darwin
          # macOS arm64
          - os: macos-14
            target: aarch64-apple-darwin
          # Windows x86_64
          - os: windows-latest
            target: x86_64-pc-windows-msvc

    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Download version artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: dev-version
          path: version-artifact
        continue-on-error: true

      - name: Resolve dev version
        shell: bash
        run: |
          if [ -f version-artifact/dev_version.txt ]; then
            DEV_VERSION=$(cat version-artifact/dev_version.txt)
            CARGO_VERSION=$(cat version-artifact/cargo_version.txt)
          else
            DEV_VERSION="${{ needs.version.outputs.dev_version }}"
            CARGO_VERSION="${{ needs.version.outputs.cargo_version }}"
          fi
          echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV"
          echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV"

      - name: Install protoc
        if: runner.os != 'Linux'
        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b  # v3.0.0
        with:
          version: ${{ env.PROTOC_VERSION }}
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Inject dev version
        shell: bash
        env:
          DEV_VERSION: ${{ needs.version.outputs.dev_version }}
          CARGO_VERSION: ${{ needs.version.outputs.cargo_version }}
        run: |
          python3 -c "
          import re, os, pathlib
          p = pathlib.Path('pyproject.toml')
          text = p.read_text()
          v = os.environ['DEV_VERSION']
          text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1)
          p.write_text(text)
          "
          python3 -c "
          import re, os, pathlib
          p = pathlib.Path('rust/Cargo.toml')
          text = p.read_text()
          v = os.environ['CARGO_VERSION']
          text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE)
          p.write_text(text)
          "
          cargo generate-lockfile --manifest-path rust/Cargo.toml
          grep '^version' pyproject.toml rust/Cargo.toml

      - name: Verify version injection
        shell: bash
        run: |
          python3 -c "
          import tomllib, os
          with open('pyproject.toml', 'rb') as f:
              v = tomllib.load(f)['project']['version']
          expected = os.environ['DEV_VERSION']
          assert v == expected, f'Expected {expected}, got {v}'
          print(f'Version injection verified: {v}')
          "

      - name: Build wheels
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1.50.1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist
          manylinux: ${{ matrix.manylinux || 'auto' }}
          before-script-linux: |
            TARGET="${{ matrix.target }}"
            PROTOC_VERSION="${{ env.PROTOC_VERSION }}"
            case "$TARGET" in
              x86_64-*)
                PROTOC_ARCH="linux-x86_64"
                PROTOC_SHA256="0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f"
                ;;
              aarch64-*)
                PROTOC_ARCH="linux-aarch_64"
                PROTOC_SHA256="1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a"
                ;;
              *)
                echo "Unsupported target: $TARGET" >&2
                exit 1
                ;;
            esac
            PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip"
            curl -fsSLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}"
            echo "${PROTOC_SHA256}  ${PROTOC_ZIP}" | sha256sum -c -
            python3 -m zipfile -e "${PROTOC_ZIP}" /usr/local
            chmod +x /usr/local/bin/protoc
            rm "${PROTOC_ZIP}"

      - name: Validate wheel platform tags (x86_64 cross-compile)
        if: matrix.target == 'x86_64-apple-darwin'
        shell: bash
        run: |
          echo "Validating wheel platform tags for x86_64 cross-compile..."
          bad=0
          for wheel in dist/*.whl; do
            echo "  $wheel"
            if [[ "$wheel" != *"x86_64"* ]]; then
              echo "ERROR: expected x86_64 platform tag but got: $wheel"
              bad=1
            fi
          done
          if [ "$bad" -ne 0 ]; then
            echo "One or more wheels have incorrect platform tags."
            exit 1
          fi
          echo "All wheels have correct macosx_*_x86_64 platform tags."

      - name: Upload wheels
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: wheels-${{ matrix.target }}
          path: dist/*.whl

  build-sdist:
    needs: version
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Download version artifact
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: dev-version
          path: version-artifact
        continue-on-error: true

      - name: Resolve dev version
        run: |
          if [ -f version-artifact/dev_version.txt ]; then
            DEV_VERSION=$(cat version-artifact/dev_version.txt)
            CARGO_VERSION=$(cat version-artifact/cargo_version.txt)
          else
            DEV_VERSION="${{ needs.version.outputs.dev_version }}"
            CARGO_VERSION="${{ needs.version.outputs.cargo_version }}"
          fi
          echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV"
          echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV"

      - name: Inject dev version
        env:
          DEV_VERSION: ${{ needs.version.outputs.dev_version }}
          CARGO_VERSION: ${{ needs.version.outputs.cargo_version }}
        run: |
          python3 -c "
          import re, os, pathlib
          p = pathlib.Path('pyproject.toml')
          text = p.read_text()
          v = os.environ['DEV_VERSION']
          text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1)
          p.write_text(text)
          "
          python3 -c "
          import re, os, pathlib
          p = pathlib.Path('rust/Cargo.toml')
          text = p.read_text()
          v = os.environ['CARGO_VERSION']
          text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE)
          p.write_text(text)
          "
          cargo generate-lockfile --manifest-path rust/Cargo.toml
          grep '^version' pyproject.toml rust/Cargo.toml

      - name: Verify version injection
        run: |
          python3 -c "
          import tomllib, os
          with open('pyproject.toml', 'rb') as f:
              v = tomllib.load(f)['project']['version']
          expected = os.environ['DEV_VERSION']
          assert v == expected, f'Expected {expected}, got {v}'
          print(f'Version injection verified: {v}')
          "

      - name: Build sdist
        uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad  # v1.50.1
        with:
          command: sdist
          args: --out dist

      - name: Upload sdist
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: sdist
          path: dist/*.tar.gz

  # ---------------------------------------------------------------------------
  # Stage 3: Publish to Artifact Registry
  # ---------------------------------------------------------------------------
  publish:
    needs: [build-wheels, build-sdist]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      id-token: write
    steps:
      - name: Download all artifacts
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          path: dist
          pattern: '{wheels-*,sdist}'
          merge-multiple: true

      - name: Validate artifact counts
        run: |
          echo "Artifacts to publish:"
          ls -la dist/
          whl_count=$(ls dist/*.whl 2>/dev/null | wc -l)
          sdist_count=$(ls dist/*.tar.gz 2>/dev/null | wc -l)
          echo "Wheels: $whl_count, Sdists: $sdist_count"
          # 7 platforms × 1 abi3 wheel (cp310-abi3, covers Python 3.10+) = 7 wheels
          if [ "$whl_count" -lt 7 ]; then
            echo "::error::expected at least 7 wheels but got $whl_count — one or more matrix legs may be missing"
            exit 1
          fi
          if [ "$sdist_count" -lt 1 ]; then
            echo "::error::sdist missing"
            exit 1
          fi

      - name: Authenticate to Google Cloud
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3.0.0
        with:
          workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ vars.GCP_SERVICE_ACCOUNT }}

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.12"

      - name: Install twine
        run: pip install twine==6.1.0

      - name: Configure Artifact Registry credentials
        run: |
          pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2

      - name: Validate wheel metadata
        run: twine check dist/*

      - name: Upload to Artifact Registry
        run: |
          twine upload \
            --skip-existing \
            --repository-url "https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/" \
            dist/*

  # ---------------------------------------------------------------------------
  # Stage 4: Smoke tests against the just-published wheel (informational only)
  # ---------------------------------------------------------------------------
  smoke-tests:
    name: Smoke tests (informational)
    needs: publish
    if: success()
    runs-on: ubuntu-latest
    timeout-minutes: 20
    continue-on-error: true
    permissions:
      contents: read
    env:
      PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.12"

      - name: Download linux x86_64 wheel
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: wheels-x86_64-unknown-linux-gnu
          path: dist

      - name: Install wheel and test deps
        run: |
          set -euo pipefail
          wheel=$(ls dist/*.whl | head -n1)
          echo "Installing ${wheel}"
          pip install --upgrade pip
          pip install "${wheel}" pytest pytest-asyncio python-dotenv

      - name: Run smoke suite
        id: smoke
        continue-on-error: true
        run: |
          # The source `pinecone/` directory from actions/checkout shadows the
          # wheel-installed package on sys.path (cwd is on the path). The source
          # tree has `pinecone/grpc/__init__.py` but not the compiled
          # `_grpc.abi3.so` extension, which only ships inside the wheel — so
          # any test that constructs a GrpcIndex would crash with
          # `ModuleNotFoundError: No module named 'pinecone._grpc'`. Smoke runs
          # the published wheel only, so we can safely delete the source.
          rm -rf pinecone rust target
          pytest tests/smoke/ \
            -v -s -rs \
            --ignore=tests/smoke/test_pod_collections_sync.py \
            --ignore=tests/smoke/test_pod_collections_async.py \
            --junitxml=smoke-results.xml

      - name: Append summary
        if: always()
        run: |
          if [ "${{ steps.smoke.outcome }}" = "success" ]; then
            echo "## Smoke tests: PASSED" >> "$GITHUB_STEP_SUMMARY"
          else
            echo "## Smoke tests: FAILED (informational — does not block publish)" >> "$GITHUB_STEP_SUMMARY"
            echo "See the 'Run smoke suite' step output and the smoke-results artifact." >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Orphan cleanup
        if: always()
        continue-on-error: true
        run: |
          python tests/smoke/scripts/cleanup_orphans.py || true

      - name: Upload smoke results
        if: always()
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f  # v7.0.0
        with:
          name: smoke-results
          path: smoke-results.xml

  notify-smoke-failure:
    name: Notify on smoke regression
    runs-on: ubuntu-latest
    needs: smoke-tests
    if: needs.smoke-tests.result == 'failure'
    permissions:
      issues: write
    steps:
      - name: Open smoke-regression issue
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          body=$(printf '%s\n' \
            "## Smoke tests regressed (informational — publish succeeded)" \
            "" \
            "**Workflow run:** ${run_url}" \
            "**Triggered by:** \`${{ github.event_name }}\`" \
            "**Ref:** \`${{ github.ref }}\`" \
            "**Commit:** \`${{ github.sha }}\`" \
            "" \
            "The smoke job ran against the just-published dev wheel and reported FAILED." \
            "Publish completed normally; this is a follow-up signal, not a build failure." \
            "" \
            "Download the \`smoke-results\` artifact from the workflow run for the JUnit report.")
          gh issue create \
            --repo "${{ github.repository }}" \
            --title "[smoke regression] $(date -u +%Y-%m-%d) — dev-publish smoke run failed" \
            --body "$body"

  # ---------------------------------------------------------------------------
  # Failure notification: open a GitHub issue when any job fails
  # ---------------------------------------------------------------------------
  notify-failure:
    name: Notify on failure
    runs-on: ubuntu-latest
    needs: [check-changes, version, build-wheels, build-sdist, publish]
    if: >-
      always() &&
      (needs.check-changes.result == 'failure' ||
       needs.version.result == 'failure' ||
       needs.build-wheels.result == 'failure' ||
       needs.build-sdist.result == 'failure' ||
       needs.publish.result == 'failure')
    permissions:
      issues: write
    steps:
      - name: Open failure issue
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          body=$(printf '%s\n' \
            "## Dev Build & Publish failed" \
            "" \
            "**Workflow run:** ${run_url}" \
            "**Triggered by:** \`${{ github.event_name }}\`" \
            "**Ref:** \`${{ github.ref }}\`" \
            "**Commit:** \`${{ github.sha }}\`" \
            "" \
            "| Job | Result |" \
            "|-----|--------|" \
            "| check-changes | ${{ needs.check-changes.result }} |" \
            "| version | ${{ needs.version.result }} |" \
            "| build-wheels | ${{ needs.build-wheels.result }} |" \
            "| build-sdist | ${{ needs.build-sdist.result }} |" \
            "| publish | ${{ needs.publish.result }} |")
          gh issue create \
            --repo "${{ github.repository }}" \
            --title "[CI] Dev Build & Publish failed — $(date -u +%Y-%m-%d)" \
            --body "$body"