add repo ingest as analysis

Author: cyclotruc

app/main.py

@@ -3,7 +3,7 @@
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from pathlib import Path
-from app.routes import install, actions
+from app.routes import install, actions, smart_ingest_route
 from app.services.actions_loader import actions_loader
 from api_analytics.fastapi import Analytics
 from fastapi_mcp import FastApiMCP
@@ -28,6 +28,7 @@
 # Include routers
 app.include_router(install.router)
 app.include_router(actions.router)
+app.include_router(smart_ingest_route.router)
 
 @app.get("/favicon.ico", operation_id="get_favicon")
 async def favicon():

app/routes/smart_ingest_route.py

@@ -0,0 +1,92 @@
+"""
+Simple route for smart ingest functionality.
+"""
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Optional
+from app.services.smart_ingest import use_gitingest, smart_ingest
+
+router = APIRouter(prefix="/api", tags=["smart_ingest"])
+
+
+class IngestRequest(BaseModel):
+    repo_url: str
+
+
+class IngestResponse(BaseModel):
+    success: bool
+    context_size: int
+    message: str
+
+
+class AnalyzeRequest(BaseModel):
+    context: str
+    user_prompt: Optional[str] = "Analyze this repository and provide a comprehensive overview"
+
+
+class AnalyzeResponse(BaseModel):
+    success: bool
+    analysis: str
+
+
+@router.post("/ingest", response_model=IngestResponse)
+async def ingest_repository(request: IngestRequest):
+    """
+    Ingest a repository and return the context.
+    """
+    try:
+        context = await use_gitingest(request.repo_url)
+        return IngestResponse(
+            success=True,
+            context_size=len(context),
+            message=context
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/analyze", response_model=AnalyzeResponse)
+async def analyze_context(request: AnalyzeRequest):
+    """
+    Analyze the provided context using OpenAI.
+    """
+    try:
+        result = smart_ingest(request.context, request.user_prompt)
+        return AnalyzeResponse(
+            success=True,
+            analysis=result.get("response", "")
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+# Keep the combined endpoint for backward compatibility
+class SmartIngestRequest(BaseModel):
+    repo_url: str
+    user_prompt: Optional[str] = "Analyze this repository and provide a comprehensive overview"
+
+
+class SmartIngestResponse(BaseModel):
+    success: bool
+    analysis: str
+
+
+@router.post("/smart_ingest", response_model=SmartIngestResponse)
+async def analyze_repository(request: SmartIngestRequest):
+    """
+    Analyze a repository using smart ingest (combined endpoint).
+    """
+    try:
+        # Step 1: Ingest the repository (async)
+        context = await use_gitingest(request.repo_url)
+        
+        # Step 2: Send to OpenAI (still sync, but that's ok)
+        result = smart_ingest(context, request.user_prompt)
+        
+        return SmartIngestResponse(
+            success=True,
+            analysis=result.get("response", "")
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

app/services/smart_ingest.py

@@ -0,0 +1,138 @@
+"""
+Functions for ingesting repositories and sending context to OpenAI API.
+"""
+
+import httpx
+from typing import Optional, Dict, Any
+from dotenv import load_dotenv
+import os
+from gitingest import ingest_async
+
+# Load environment variables from .env file
+load_dotenv()
+
+
+async def use_gitingest(url: str, context_size: int = 50000) -> str:
+    """
+    Ingest a repository using gitingest and trim to specified token size.
+    
+    Args:
+        url: Repository URL to ingest
+        context_size: Maximum context size in tokens (default ~50k tokens)
+    
+    Returns:
+        String containing the repository context, trimmed to specified size
+    """
+    # Ingest the repository
+    summary, tree, content = await ingest_async(
+        url,
+        max_file_size=512000,
+        include_patterns=None,
+        exclude_patterns=None
+    )
+    
+    # Combine into single context
+    full_context = f"{summary}\n\n{tree}\n\n{content}"
+    
+    # Approximate token count (roughly 4 chars per token)
+    # Trim to specified context size
+    max_chars = context_size * 4
+    if len(full_context) > max_chars:
+        full_context = full_context[:max_chars]
+        # Add ellipsis to indicate truncation
+        full_context += "\n\n... (context truncated)"
+    
+    return full_context
+
+
+def smart_ingest(
+    context: str, 
+    user_prompt: str = "Analyze this repository and provide insights",
+    api_key: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Send the ingested repository context to OpenAI API with a system prompt.
+    
+    Args:
+        context: The "big fat context" from use_git_ingest function
+        user_prompt: The user's question or request about the repository
+        api_key: Optional OpenAI API key (defaults to env var OPENAI_API_KEY)
+    
+    Returns:
+        Dictionary containing OpenAI's response and metadata
+    
+    Raises:
+        Exception: If the API call fails
+    """
+    # Get API key from environment if not provided
+    if not api_key:
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            raise ValueError("OPENAI_API_KEY not found in environment variables")
+    
+    # System prompt for repository analysis
+    system_prompt = """You are an expert code analyst and software architect. 
+You have been given the complete context of a repository including its structure and file contents.
+Analyze the repository thoroughly and provide insights based on the user's request.
+Focus on:
+- Code quality and architecture
+- Potential improvements
+- Security considerations
+- Documentation completeness
+- Dependencies and technical debt
+Be specific and provide actionable recommendations."""
+    
+    # Prepare messages for OpenAI
+    messages = [
+        {
+            "role": "system",
+            "content": system_prompt
+        },
+        {
+            "role": "user",
+            "content": f"{user_prompt}\n\n{context}"
+        }
+    ]
+    
+    # OpenAI API endpoint
+    url = "https://api.openai.com/v1/chat/completions"
+    
+    # Headers for the API request
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    
+    # Request body
+    data = {
+        "model": "gpt-4o-mini",  # Using GPT-4o-mini for cost efficiency
+        "messages": messages,
+        "temperature": 0.3,  # Lower temperature for more focused analysis
+        "max_tokens": 4096
+    }
+    
+    try:
+        # Make the API call
+        with httpx.Client(timeout=60.0) as client:
+            response = client.post(url, json=data, headers=headers)
+            response.raise_for_status()
+            
+            result = response.json()
+            
+            # Extract the response
+            choice = result.get("choices", [{}])[0]
+            message = choice.get("message", {})
+            
+            return {
+                "success": True,
+                "response": message.get("content", ""),
+                "model": result.get("model"),
+                "usage": result.get("usage", {}),
+                "finish_reason": choice.get("finish_reason")
+            }
+            
+    except httpx.HTTPStatusError as e:
+        error_detail = e.response.text if e.response else str(e)
+        raise Exception(f"OpenAI API error: {e.response.status_code} - {error_detail}")
+    except Exception as e:
+        raise Exception(f"Failed to send context to OpenAI: {str(e)}")