Major code reorganization: modular structure with separate folders

Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com>

Author: Copilot

ai/__init__.py

@@ -0,0 +1,18 @@
+"""
+AI and analysis modules.
+"""
+from .analyzer import (
+    analyze_local_path_background,
+    analyze_local_path_sync,
+    search_semantic,
+    call_coding_model,
+    llama_index_retrieve_documents,
+)
+
+__all__ = [
+    'analyze_local_path_background',
+    'analyze_local_path_sync',
+    'search_semantic',
+    'call_coding_model',
+    'llama_index_retrieve_documents',
+]

analyzer.py ai/analyzer.pyanalyzer.py renamed to ai/analyzer.py

@@ -14,7 +14,7 @@
 from db import store_file, needs_reindex, set_project_metadata_batch, get_project_metadata
 from external_api import get_embedding_for_text, call_coding_api
 from llama_index.core import Document
-from logger import get_logger
+from utils.logger import get_logger
 from smart_chunker import smart_chunk
 import logging
 
@@ -578,18 +578,7 @@ def llama_index_retrieve_documents(query: str, database_path: str, top_k: int =
     """
     Return llama_index.core.Document objects for the top_k matching chunks using sqlite-vector.
     """
-    q_emb = get_embedding_for_text(query)
-    if not q_emb:
-        return []
-
-    rows = _search_vectors(database_path, q_emb, top_k=top_k)
-    docs: List[Document] = []
-    for r in rows:
-        fid = r.get("file_id")
-        path = r.get("path")
-        chunk_idx = r.get("chunk_index", 0)
-        score = r.get("score", 0.0)
-        chunk_text = _get_chunk_text(database_path, fid, chunk_idx) or ""
-        doc = Document(text=chunk_text, extra_info={"path": path, "file_id": fid, "chunk_index": chunk_idx, "score": score})
-        docs.append(doc)
-    return docs
+    from .llama_integration import llama_index_retrieve_documents as _llama_retrieve
+    return _llama_retrieve(query, database_path, top_k, 
+                          search_func=_search_vectors, 
+                          get_chunk_func=_get_chunk_text)

ai/llama_integration.py

@@ -0,0 +1,45 @@
+"""
+LlamaIndex integration for document retrieval.
+"""
+from typing import List
+from llama_index.core import Document
+
+from external_api import get_embedding_for_text
+from utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def llama_index_retrieve_documents(query: str, database_path: str, top_k: int = 5, 
+                                   search_func=None, get_chunk_func=None) -> List[Document]:
+    """
+    Return llama_index.core.Document objects for the top_k matching chunks using sqlite-vector.
+    
+    Args:
+        query: Search query text
+        database_path: Path to project database
+        top_k: Number of results to return
+        search_func: Function to search vectors (injected from analyzer)
+        get_chunk_func: Function to get chunk text (injected from analyzer)
+    
+    Returns:
+        List of Document objects with chunk text and metadata
+    """
+    if search_func is None or get_chunk_func is None:
+        raise ValueError("search_func and get_chunk_func must be provided")
+    
+    q_emb = get_embedding_for_text(query)
+    if not q_emb:
+        return []
+
+    rows = search_func(database_path, q_emb, top_k=top_k)
+    docs: List[Document] = []
+    for r in rows:
+        fid = r.get("file_id")
+        path = r.get("path")
+        chunk_idx = r.get("chunk_index", 0)
+        score = r.get("score", 0.0)
+        chunk_text = get_chunk_func(database_path, fid, chunk_idx) or ""
+        doc = Document(text=chunk_text, extra_info={"path": path, "file_id": fid, "chunk_index": chunk_idx, "score": score})
+        docs.append(doc)
+    return docs

db/__init__.py

@@ -0,0 +1,68 @@
+"""
+Database operations module.
+"""
+from .operations import (
+    # Connection and initialization
+    init_db,
+    # File operations
+    store_file,
+    get_file_by_path,
+    needs_reindex,
+    list_files,
+    delete_file_by_path,
+    clear_project_data,
+    # Project registry operations
+    create_project,
+    get_project,
+    get_project_by_id,
+    list_projects,
+    update_project_status,
+    update_project_settings,
+    delete_project,
+    get_or_create_project,
+    # Metadata operations
+    set_project_metadata,
+    set_project_metadata_batch,
+    get_project_metadata,
+    get_project_stats,
+    # Chunk operations
+    insert_chunk_row_with_null_embedding,
+)
+
+from .models import (
+    CreateProjectRequest,
+    IndexProjectRequest,
+    QueryRequest,
+)
+
+__all__ = [
+    # Connection and initialization
+    'init_db',
+    # File operations
+    'store_file',
+    'get_file_by_path',
+    'needs_reindex',
+    'list_files',
+    'delete_file_by_path',
+    'clear_project_data',
+    # Project registry operations
+    'create_project',
+    'get_project',
+    'get_project_by_id',
+    'list_projects',
+    'update_project_status',
+    'update_project_settings',
+    'delete_project',
+    'get_or_create_project',
+    # Metadata operations
+    'set_project_metadata',
+    'set_project_metadata_batch',
+    'get_project_metadata',
+    'get_project_stats',
+    # Chunk operations
+    'insert_chunk_row_with_null_embedding',
+    # Models
+    'CreateProjectRequest',
+    'IndexProjectRequest',
+    'QueryRequest',
+]

db/db_task.py

@@ -0,0 +1,14 @@
+"""
+Database task class for queued write operations.
+"""
+import threading
+
+
+class _DBTask:
+    """Internal task class for queuing database write operations."""
+    def __init__(self, sql, params):
+        self.sql = sql
+        self.params = params
+        self.event = threading.Event()
+        self.rowid = None
+        self.exception = None

models.py db/models.pymodels.py renamed to db/models.py

@@ -3,12 +3,13 @@
 from typing import Any, Dict, List, Optional
 from functools import lru_cache
 
-from config import CFG  # config (keeps chunk_size etc if needed)
+from utils.config import CFG  # config (keeps chunk_size etc if needed)
 import atexit
 import threading
 import queue
-from logger import get_logger
+from utils.logger import get_logger
 from cache import project_cache, stats_cache, file_cache
+from .db_task import _DBTask
 
 _LOG = get_logger(__name__)
 
@@ -20,13 +21,6 @@
 _WRITERS = {}
 _WRITERS_LOCK = threading.Lock()
 
-class _DBTask:
-    def __init__(self, sql, params):
-        self.sql = sql
-        self.params = params
-        self.event = threading.Event()
-        self.rowid = None
-        self.exception = None
 
 class DBWriter:
     def __init__(self, database_path, timeout_seconds=30):

db.py db/operations.pydb.py renamed to db/operations.py

@@ -0,0 +1,8 @@
+"""
+API endpoints module.
+"""
+from .project_endpoints import router as project_router
+from .query_endpoints import router as query_router
+from .web_endpoints import router as web_router
+
+__all__ = ['project_router', 'query_router', 'web_router']

endpoints/__init__.py

@@ -0,0 +1,179 @@
+"""
+Project management API endpoints.
+"""
+from fastapi import APIRouter, Request, BackgroundTasks
+from fastapi.responses import JSONResponse
+import os
+from datetime import datetime
+
+from db import (
+    get_project_by_id, list_projects,
+    update_project_status, delete_project, get_or_create_project,
+    CreateProjectRequest, IndexProjectRequest
+)
+from ai.analyzer import analyze_local_path_background
+from utils.logger import get_logger
+from utils.config import CFG
+from .rate_limiter import indexing_limiter
+
+logger = get_logger(__name__)
+router = APIRouter(prefix="/api", tags=["projects"])
+
+MAX_FILE_SIZE = int(CFG.get("max_file_size", 200000))
+
+
+def _get_client_ip(request: Request) -> str:
+    """Get client IP address from request."""
+    forwarded = request.headers.get("X-Forwarded-For")
+    if forwarded:
+        return forwarded.split(",")[0].strip()
+    return request.client.host if request.client else "unknown"
+
+
+@router.post("/projects", summary="Create or get a project")
+def api_create_project(request: CreateProjectRequest):
+    """
+    Create or get a project with per-project database.
+    
+    - **path**: Absolute path to project directory (required)
+    - **name**: Optional project name (defaults to directory name)
+    
+    Returns project metadata including:
+    - **id**: Unique project identifier
+    - **database_path**: Path to project's SQLite database
+    - **status**: Current project status
+    """
+    try:
+        project = get_or_create_project(request.path, request.name)
+        return JSONResponse(project)
+    except ValueError as e:
+        # ValueError is expected for invalid inputs, safe to show message
+        logger.warning(f"Validation error creating project: {e}")
+        return JSONResponse({"error": "Invalid project path"}, status_code=400)
+    except RuntimeError as e:
+        # RuntimeError may contain sensitive details, use generic message
+        logger.error(f"Runtime error creating project: {e}")
+        return JSONResponse({"error": "Database operation failed"}, status_code=500)
+    except Exception as e:
+        logger.exception(f"Unexpected error creating project: {e}")
+        return JSONResponse({"error": "Internal server error"}, status_code=500)
+
+
+@router.get("/projects", summary="List all projects")
+def api_list_projects():
+    """
+    List all registered projects.
+    
+    Returns array of project objects with metadata:
+    - **id**: Unique project identifier  
+    - **name**: Project name
+    - **path**: Project directory path
+    - **status**: Current status (created, indexing, ready, error)
+    - **last_indexed_at**: Last indexing timestamp
+    """
+    try:
+        projects = list_projects()
+        return JSONResponse(projects)
+    except Exception as e:
+        logger.exception(f"Error listing projects: {e}")
+        return JSONResponse({"error": "Failed to list projects"}, status_code=500)
+
+
+@router.get("/projects/{project_id}", summary="Get project by ID")
+def api_get_project(project_id: str):
+    """
+    Get project details by ID.
+    
+    - **project_id**: Unique project identifier
+    
+    Returns project metadata or 404 if not found.
+    """
+    try:
+        project = get_project_by_id(project_id)
+        if not project:
+            return JSONResponse({"error": "Project not found"}, status_code=404)
+        return JSONResponse(project)
+    except Exception as e:
+        logger.exception(f"Error getting project: {e}")
+        return JSONResponse({"error": "Failed to retrieve project"}, status_code=500)
+
+
+@router.delete("/projects/{project_id}", summary="Delete a project")
+def api_delete_project(project_id: str):
+    """
+    Delete a project and its database.
+    
+    - **project_id**: Unique project identifier
+    
+    Permanently removes the project and all indexed data.
+    Returns 404 if project not found.
+    """
+    try:
+        delete_project(project_id)
+        return JSONResponse({"success": True})
+    except ValueError as e:
+        logger.warning(f"Project not found for deletion: {e}")
+        return JSONResponse({"error": "Project not found"}, status_code=404)
+    except Exception as e:
+        logger.exception(f"Error deleting project: {e}")
+        return JSONResponse({"error": "Failed to delete project"}, status_code=500)
+
+
+@router.post("/projects/index", tags=["indexing"], summary="Index a project")
+def api_index_project(http_request: Request, request: IndexProjectRequest, background_tasks: BackgroundTasks):
+    """
+    Index or re-index a project in the background.
+    
+    - **project_id**: Unique project identifier
+    
+    Starts background indexing process:
+    - Scans project directory for code files
+    - Generates embeddings for semantic search
+    - Uses incremental indexing (skips unchanged files)
+    
+    Rate limit: 10 requests per minute per IP.
+    
+    Returns immediately with status "indexing".
+    Poll project status to check completion.
+    """
+    # Rate limiting for indexing operations (more strict)
+    client_ip = _get_client_ip(http_request)
+    allowed, retry_after = indexing_limiter.is_allowed(client_ip)
+    if not allowed:
+        return JSONResponse(
+            {"error": "Rate limit exceeded for indexing", "retry_after": retry_after},
+            status_code=429,
+            headers={"Retry-After": str(retry_after)}
+        )
+    
+    try:
+        project = get_project_by_id(request.project_id)
+        if not project:
+            return JSONResponse({"error": "Project not found"}, status_code=404)
+        
+        project_path = project["path"]
+        db_path = project["database_path"]
+        
+        if not os.path.exists(project_path):
+            return JSONResponse({"error": "Project path does not exist"}, status_code=400)
+        
+        # Update status to indexing
+        update_project_status(request.project_id, "indexing")
+        
+        # Start background indexing
+        venv_path = CFG.get("venv_path")
+        
+        def index_callback():
+            try:
+                analyze_local_path_background(project_path, db_path, venv_path, MAX_FILE_SIZE, CFG)
+                update_project_status(request.project_id, "ready", datetime.utcnow().isoformat())
+            except Exception as e:
+                update_project_status(request.project_id, "error")
+                raise
+        
+        background_tasks.add_task(index_callback)
+        
+        return JSONResponse({"status": "indexing", "project_id": request.project_id})
+    except Exception as e:
+        logger.exception(f"Error starting project indexing: {e}")
+        return JSONResponse({"error": "Failed to start indexing"}, status_code=500)