feat: add docs-ingestion and docs-embeddings pipelines

.github/workflows/docs-ingestion-ci.yml

@@ -0,0 +1,327 @@
+name: Docs Ingestion CI
+
+on:
+  pull_request:
+    branches: [main, staging]
+    paths:
+      - 'docs-ingestion/**'
+      - 'docs-embeddings/**'
+      - 'api-references/**'
+      - 'content/**'
+  push:
+    branches: [main, staging]
+    paths:
+      - 'docs-ingestion/**'
+      - 'docs-embeddings/**'
+      - 'api-references/**'
+      - 'content/**'
+
+jobs:
+  build-and-test:
+    name: Build & Test
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: docs-ingestion
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+          cache-dependency-path: docs-ingestion/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --immutable
+
+      - name: Security audit
+        run: yarn npm audit --severity moderate
+        continue-on-error: true
+
+      - name: Build
+        run: yarn build
+
+      - name: Normalize MDX (required by integration tests)
+        run: yarn normalize-mdx
+
+      # ── E. Test suite ──
+      - name: Run tests
+        run: yarn test
+
+  normalize-api-specs:
+    name: API Spec Normalization
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: docs-ingestion
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+          cache-dependency-path: docs-ingestion/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --immutable
+
+      # ── A. Normalization run ──
+      - name: Run API spec normalization
+        run: yarn normalize-api-specs
+
+      - name: Verify output directory exists
+        run: |
+          if [ ! -d "../.api-reference-normalized" ]; then
+            echo "FAIL: .api-reference-normalized/ directory does not exist"
+            exit 1
+          fi
+          echo "PASS: Directory exists"
+
+      - name: Verify file count
+        run: |
+          count=$(find ../.api-reference-normalized -name '*.md' | wc -l | tr -d ' ')
+          echo "Found $count normalized files"
+          if [ "$count" -lt 200 ]; then
+            echo "FAIL: Expected at least 200 files, got $count"
+            exit 1
+          fi
+          echo "PASS: File count ($count) >= 200"
+
+      # ── B. Determinism check ──
+      - name: Copy first run output
+        run: cp -r ../.api-reference-normalized /tmp/api-ref-norm-run1
+
+      - name: Run normalization again
+        run: yarn normalize-api-specs
+
+      - name: Verify determinism
+        run: |
+          diff_output=$(diff -r ../.api-reference-normalized /tmp/api-ref-norm-run1 2>&1) || true
+          if [ -n "$diff_output" ]; then
+            echo "FAIL: Normalization is not deterministic:"
+            echo "$diff_output" | head -20
+            exit 1
+          fi
+          echo "PASS: Output is deterministic"
+
+      # ── C. Token limit compliance ──
+      - name: Check token limits
+        run: yarn check-token-limits
+
+      # ── F. Git ignored state check ──
+      - name: Verify .api-reference-normalized is gitignored
+        run: |
+          cd ..
+          if git ls-files --error-unmatch .api-reference-normalized/ 2>/dev/null; then
+            echo "FAIL: .api-reference-normalized/ is tracked by git"
+            exit 1
+          fi
+          echo "PASS: .api-reference-normalized/ is not tracked"
+
+      - name: Verify .docs-normalized is gitignored
+        run: |
+          cd ..
+          if git ls-files --error-unmatch .docs-normalized/ 2>/dev/null; then
+            echo "FAIL: .docs-normalized/ is tracked by git"
+            exit 1
+          fi
+          echo "PASS: .docs-normalized/ is not tracked"
+
+  ingestion-smoke-test:
+    name: Ingestion Smoke Test
+    runs-on: ubuntu-latest
+    needs: [build-and-test, normalize-api-specs]
+    defaults:
+      run:
+        working-directory: docs-ingestion
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+          cache-dependency-path: docs-ingestion/yarn.lock
+
+      - name: Install dependencies
+        run: yarn install --immutable
+
+      - name: Build
+        run: yarn build
+
+      # Generate the normalized API specs (required for smoke test)
+      - name: Normalize API specs
+        run: yarn normalize-api-specs
+
+      # Normalize MDX if content/ exists
+      - name: Normalize MDX (if content exists)
+        run: |
+          if [ -d "../content" ]; then
+            yarn normalize-mdx || exit 1
+          else
+            echo "No content/ directory — skipping MDX normalization"
+          fi
+
+      # ── D. Ingestion smoke test ──
+      - name: Run ingestion smoke test
+        run: yarn smoke-test-ingestion
+
+  embedding-dry-run:
+    name: Embedding Dry Run
+    runs-on: ubuntu-latest
+    needs: [ingestion-smoke-test]
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+          cache-dependency-path: |
+            docs-ingestion/yarn.lock
+            docs-embeddings/yarn.lock
+
+      # Build ingestion pipeline and produce chunks.json
+      - name: Install ingestion dependencies
+        working-directory: docs-ingestion
+        run: yarn install --immutable
+
+      - name: Build ingestion
+        working-directory: docs-ingestion
+        run: yarn build
+
+      - name: Normalize API specs
+        working-directory: docs-ingestion
+        run: yarn normalize-api-specs
+
+      - name: Normalize MDX (if content exists)
+        working-directory: docs-ingestion
+        run: |
+          if [ -d "../content" ]; then
+            yarn normalize-mdx || exit 1
+          else
+            echo "No content/ directory — skipping MDX normalization"
+          fi
+
+      - name: Run ingestion pipeline
+        working-directory: docs-ingestion
+        run: node dist/index.js
+
+      - name: Verify chunks.json exists
+        run: |
+          if [ ! -f "docs-ingestion/output/chunks.json" ]; then
+            echo "FAIL: chunks.json not produced"
+            exit 1
+          fi
+          echo "PASS: chunks.json exists"
+
+      # Build embeddings pipeline and run dry-run
+      - name: Install embedding dependencies
+        working-directory: docs-embeddings
+        run: yarn install --immutable
+
+      - name: Build embeddings
+        working-directory: docs-embeddings
+        run: yarn build
+
+      # ── E. Embedding dry run ──
+      - name: Run embedding dry run
+        working-directory: docs-embeddings
+        env:
+          DRY_RUN: 'true'
+          INGESTION_OUTPUT_PATH: ${{ github.workspace }}/docs-ingestion/output/chunks.json
+        run: node dist/index.js --dry-run
+
+  # ── Deploy: update knowledge base (main only) ──
+  deploy-knowledge-base:
+    name: Deploy Knowledge Base
+    runs-on: ubuntu-latest
+    needs: [embedding-dry-run]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '20'
+          cache: 'yarn'
+          cache-dependency-path: |
+            docs-ingestion/yarn.lock
+            docs-embeddings/yarn.lock
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ap-south-1
+
+      # ── Build ingestion pipeline ──
+      - name: Install ingestion dependencies
+        working-directory: docs-ingestion
+        run: yarn install --immutable
+
+      - name: Build ingestion
+        working-directory: docs-ingestion
+        run: yarn build
+
+      - name: Normalize API specs
+        working-directory: docs-ingestion
+        run: yarn normalize-api-specs
+
+      - name: Normalize MDX
+        working-directory: docs-ingestion
+        run: yarn normalize-mdx
+
+      - name: Run ingestion pipeline
+        working-directory: docs-ingestion
+        run: node dist/index.js
+
+      # ── Upload content to S3 ──
+      - name: Upload content to S3
+        working-directory: docs-ingestion
+        env:
+          CONTENT_BUCKET_NAME: ${{ secrets.CONTENT_BUCKET_NAME }}
+        run: node dist/upload-content.js
+
+      # ── Build and run embedding sync ──
+      - name: Install embedding dependencies
+        working-directory: docs-embeddings
+        run: yarn install --immutable
+
+      - name: Build embeddings
+        working-directory: docs-embeddings
+        run: yarn build
+
+      - name: Run embedding sync
+        working-directory: docs-embeddings
+        env:
+          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+          PINECONE_INDEX: ${{ secrets.PINECONE_INDEX }}
+          CONTENT_BUCKET_NAME: ${{ secrets.CONTENT_BUCKET_NAME }}
+          INGESTION_OUTPUT_PATH: ${{ github.workspace }}/docs-ingestion/output/chunks.json
+        run: node dist/index.js

.gitignore

@@ -0,0 +1,14 @@
+# Normalized MDX output (generated by docs-ingestion/normalize-mdx)
+.docs-normalized/
+
+# Normalized API spec output (generated by docs-ingestion/normalize-api-specs)
+.api-reference-normalized/
+
+# Ruflo
+.ruflo/
+
+# Claude Code
+.claude-flow/
+.mcp.json
+.claude
+.swarm