🔨 improve CI/CD and code quality

.github/workflows/agents_validate.yaml

@@ -0,0 +1,48 @@
+name: Validate AGENTS.md
+
+on:
+  push:
+    branches:
+      - '**'
+    paths:
+      - 'AGENTS.md'
+      - 'CLAUDE.md'
+      - 'scripts/validate_agents_md.py'
+      - '.github/workflows/agents_validate.yaml'
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - 'AGENTS.md'
+      - 'CLAUDE.md'
+      - 'scripts/validate_agents_md.py'
+      - '.github/workflows/agents_validate.yaml'
+  workflow_dispatch:
+
+jobs:
+  agents_validate:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Checkout submodules
+      run: |
+        git submodule init
+        git submodule update
+    - name: Read Python version
+      run: echo "PYTHON_VERSION=$(cat .python-version | tr -d '\n')" >> $GITHUB_ENV
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    - name: Install uv
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    - name: Install dependencies
+      run: |
+        uv sync
+    - name: Validate AGENTS.md
+      run: |
+        make agents_validate

.github/workflows/linter_require_ruff.yaml

@@ -15,9 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Read Python version
+        run: echo "PYTHON_VERSION=$(cat .python-version | tr -d '\n')" >> $GITHUB_ENV
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ env.PYTHON_VERSION }}
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Add uv to path

.github/workflows/nightly_tests.yaml

@@ -0,0 +1,49 @@
+name: Nightly Tests
+
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+jobs:
+  nightly_tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        submodules: 'recursive'
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Checkout submodules
+      run: |
+        git submodule init
+        git submodule update
+    - name: Read Python version
+      run: echo "PYTHON_VERSION=$(cat .python-version | tr -d '\n')" >> $GITHUB_ENV
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+    - name: Install uv
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+    - name: Install dependencies
+      run: |
+        uv sync
+    - name: Set environment variables
+      run: |
+        echo "DEV_ENV=${{ secrets.DEV_ENV }}" >> $GITHUB_ENV
+        echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV
+        echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV
+        echo "GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}" >> $GITHUB_ENV
+        echo "PERPLEXITY_API_KEY=${{ secrets.PERPLEXITY_API_KEY }}" >> $GITHUB_ENV
+        echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" >> $GITHUB_ENV
+    - name: Run slow tests
+      run: |
+        make test_slow
+    - name: Run nondeterministic tests
+      run: |
+        make test_nondeterministic
+    - name: Validate AGENTS.md
+      run: |
+        make agents_validate

.github/workflows/test_target_tests.yaml

@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         submodules: 'recursive'
         token: ${{ secrets.GITHUB_TOKEN }}
@@ -40,7 +40,7 @@ jobs:
         echo "Log Level: ${{ github.event.inputs.log_level }}"
         echo "Environment: ${{ github.event.inputs.environment }}"
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ env.PYTHON_VERSION }}
     - name: Install uv
@@ -56,12 +56,9 @@ jobs:
         echo "GROQ_API_KEY=${{ secrets.GROQ_API_KEY }}" >> $GITHUB_ENV
         echo "PERPLEXITY_API_KEY=${{ secrets.PERPLEXITY_API_KEY }}" >> $GITHUB_ENV
         echo "GEMINI_API_KEY=${{ secrets.GEMINI_API_KEY }}" >> $GITHUB_ENV
-    - name: Run tests
+    - name: Run fast tests
       run: |
-        make test
+        make test_fast
     - name: Run flaky test detection
       run: |
         make test_flaky
-    - name: Validate AGENTS.md
-      run: |
-        make agents_validate

.pre-commit-config.yaml

@@ -11,3 +11,9 @@ repos:
         language: system
         pass_filenames: false
         always_run: true
+      - id: ruff-check
+        name: Run ruff linter
+        entry: uv run ruff check
+        language: system
+        pass_filenames: false
+        always_run: true

CLAUDE.md

@@ -15,6 +15,10 @@ make all            # Run main.py with setup
 
 # Testing
 make test           # Run pytest on tests/
+make test_fast      # Run fast tests (no slow/nondeterministic)
+make test_flaky     # Repeat fast tests to detect flakiness
+make test_slow      # Run slow tests only
+make test_nondeterministic # Run nondeterministic tests only
 
 # Code Quality (run after major changes)
 make fmt            # Run black formatter + JSON formatting

Makefile

@@ -130,14 +130,39 @@ docs: ## Run docs with bun
 TEST_TARGETS = tests/
 
 ### Testing
-test: check_uv ## Run pytest tests
+test: check_uv ## Run all pytest tests
 	@echo "$(GREEN)🧪Running Target Tests...$(RESET)"
 	$(TEST) $(TEST_TARGETS)
 	@echo "$(GREEN)✅Target Tests Passed.$(RESET)"
 
-test_flaky: check_uv ## Run tests twice to detect flaky tests
+test_fast: check_uv ## Run fast tests (exclude slow/nondeterministic)
+	@echo "$(GREEN)🧪Running Fast Tests...$(RESET)"
+	$(TEST) -m "not slow and not nondeterministic" $(TEST_TARGETS)
+	@echo "$(GREEN)✅Fast Tests Passed.$(RESET)"
+
+test_slow: check_uv ## Run slow tests only
+	@echo "$(GREEN)🧪Running Slow Tests...$(RESET)"
+	@$(TEST) -m "slow" $(TEST_TARGETS); \
+	status=$$?; \
+	if [ $$status -eq 5 ]; then \
+		echo "$(YELLOW)⚠️ No slow tests collected.$(RESET)"; \
+		exit 0; \
+	fi; \
+	exit $$status
+
+test_nondeterministic: check_uv ## Run nondeterministic tests only
+	@echo "$(GREEN)🧪Running Nondeterministic Tests...$(RESET)"
+	@$(TEST) -m "nondeterministic" $(TEST_TARGETS); \
+	status=$$?; \
+	if [ $$status -eq 5 ]; then \
+		echo "$(YELLOW)⚠️ No nondeterministic tests collected.$(RESET)"; \
+		exit 0; \
+	fi; \
+	exit $$status
+
+test_flaky: check_uv ## Repeat fast tests to detect flaky tests
 	@echo "$(GREEN)🧪Running Flaky Test Detection...$(RESET)"
-	$(TEST) --count 2 -m "not slow" $(TEST_TARGETS)
+	$(TEST) --count 2 -m "not slow and not nondeterministic" $(TEST_TARGETS)
 	@echo "$(GREEN)✅Flaky Test Detection Passed.$(RESET)"
 
 

common/global_config.py

@@ -58,10 +58,10 @@ def recursive_update(default: dict, override: dict) -> dict:
         try:
             with open(config_path, "r") as file:
                 config_data = yaml.safe_load(file) or {}
-        except FileNotFoundError:
-            raise RuntimeError(f"Required config file not found: {config_path}")
+        except FileNotFoundError as e:
+            raise RuntimeError(f"Required config file not found: {config_path}") from e
         except yaml.YAMLError as e:
-            raise RuntimeError(f"Invalid YAML in {config_path}: {e}")
+            raise RuntimeError(f"Invalid YAML in {config_path}: {e}") from e
 
         # Load split YAML files from common/ directory
         reserved_filenames = {
@@ -89,7 +89,7 @@ def recursive_update(default: dict, override: dict) -> dict:
                         f"Loaded split config: {split_file.name} -> '{root_key}'"
                     )
             except yaml.YAMLError as e:
-                raise RuntimeError(f"Invalid YAML in {split_file}: {e}")
+                raise RuntimeError(f"Invalid YAML in {split_file}: {e}") from e
 
         # Load production config if in prod environment
         if os.getenv("DEV_ENV") == "prod":
@@ -108,7 +108,9 @@ def recursive_update(default: dict, override: dict) -> dict:
                         f"Production config file not found: {prod_config_path}"
                     )
                 except yaml.YAMLError as e:
-                    raise RuntimeError(f"Invalid YAML in {prod_config_path}: {e}")
+                    raise RuntimeError(
+                        f"Invalid YAML in {prod_config_path}: {e}"
+                    ) from e
 
         # Load custom local config if it exists (highest priority)
         custom_config_path = root_dir / ".global_config.yaml"
@@ -126,7 +128,7 @@ def recursive_update(default: dict, override: dict) -> dict:
             except FileNotFoundError:
                 logger.warning(f"Custom config file not found: {custom_config_path}")
             except yaml.YAMLError as e:
-                raise RuntimeError(f"Invalid YAML in {custom_config_path}: {e}")
+                raise RuntimeError(f"Invalid YAML in {custom_config_path}: {e}") from e
 
         return config_data
 
@@ -221,45 +223,52 @@ def to_dict(self) -> dict[str, Any]:
         """Convert config to dictionary."""
         return self.model_dump()
 
+    def _identify_provider(self, model_name: str) -> str:
+        """Identify the LLM provider from a model name string."""
+        name_lower = model_name.lower()
+        if "gpt" in name_lower or re.match(OPENAI_O_SERIES_PATTERN, name_lower):
+            return "openai"
+        if "claude" in name_lower or "anthropic" in name_lower:
+            return "anthropic"
+        if "groq" in name_lower:
+            return "groq"
+        if "perplexity" in name_lower:
+            return "perplexity"
+        if "gemini" in name_lower:
+            return "gemini"
+        return "unknown"
+
     def llm_api_key(self, model_name: str | None = None) -> str:
         """Returns the appropriate API key based on the model name."""
         model_identifier = model_name or self.model_name
-        if "gpt" in model_identifier.lower() or re.match(
-            OPENAI_O_SERIES_PATTERN, model_identifier.lower()
-        ):
-            return self.OPENAI_API_KEY
-        elif (
-            "claude" in model_identifier.lower()
-            or "anthropic" in model_identifier.lower()
-        ):
-            return self.ANTHROPIC_API_KEY
-        elif "groq" in model_identifier.lower():
-            return self.GROQ_API_KEY
-        elif "perplexity" in model_identifier.lower():
-            return self.PERPLEXITY_API_KEY
-        elif "gemini" in model_identifier.lower():
-            return self.GEMINI_API_KEY
-        else:
-            raise ValueError(f"No API key configured for model: {model_identifier}")
+        provider = self._identify_provider(model_identifier)
+        api_keys = {
+            "openai": self.OPENAI_API_KEY,
+            "anthropic": self.ANTHROPIC_API_KEY,
+            "groq": self.GROQ_API_KEY,
+            "perplexity": self.PERPLEXITY_API_KEY,
+            "gemini": self.GEMINI_API_KEY,
+        }
+        if provider in api_keys:
+            return api_keys[provider]
+        raise ValueError(f"No API key configured for model: {model_identifier}")
 
     def api_base(self, model_name: str) -> str:
         """Returns the Helicone link for the model.
 
         Raises:
             ValueError: If no API base is configured for the given model.
         """
-        if "gpt" in model_name.lower() or re.match(
-            OPENAI_O_SERIES_PATTERN, model_name.lower()
-        ):
-            return "https://oai.hconeai.com/v1"
-        elif "groq" in model_name.lower():
-            return "https://groq.helicone.ai/openai/v1"
-        elif "perplexity" in model_name.lower():
-            return "https://perplexity.helicone.ai"
-        elif "gemini" in model_name.lower():
-            return "https://generativelanguage.googleapis.com/v1beta/openai/"
-        else:
-            raise ValueError(f"No API base configured for model: {model_name}")
+        provider = self._identify_provider(model_name)
+        api_bases = {
+            "openai": "https://oai.hconeai.com/v1",
+            "groq": "https://groq.helicone.ai/openai/v1",
+            "perplexity": "https://perplexity.helicone.ai",
+            "gemini": "https://generativelanguage.googleapis.com/v1beta/openai/",
+        }
+        if provider in api_bases:
+            return api_bases[provider]
+        raise ValueError(f"No API base configured for model: {model_name}")
 
 
 # Load .env files before creating the config instance
@@ -276,7 +285,11 @@ def api_base(self, model_name: str) -> str:
     env_file_to_check = ".prod.env" if os.getenv("DEV_ENV") == "prod" else ".env"
     env_values = dotenv_values(root_dir / env_file_to_check)
     if not env_values:
-        warnings.warn(f"{env_file_to_check} file not found or empty", UserWarning)
+        warnings.warn(
+            f"{env_file_to_check} file not found or empty",
+            UserWarning,
+            stacklevel=2,
+        )
 
 # Create a singleton instance
 # Note: Config() loads all required fields from YAML and .env files via custom settings sources

pyproject.toml

@@ -44,8 +44,8 @@ line-length = 88
 target-version = "py312"
 
 [tool.ruff.lint]
-select = ["E", "F", "W", "I", "N", "UP"]
-ignore = ["E501", "UP015"]
+select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM"]
+ignore = ["E501", "UP015", "B008"]
 
 [tool.ty.environment]
 python-version = "3.12"
@@ -77,7 +77,6 @@ omit = [
     "docs/*",
     "init/*",
     "alembic/*",
-    "utils/llm/**"
 ]
 
 [tool.coverage.report]

pytest.ini

@@ -3,7 +3,7 @@ markers =
     slow: marks tests as slow
     nondeterministic: marks tests as nondeterministic
     slow_and_nondeterministic: marks tests as both slow and nondeterministic
-addopts = --cov-report=term-missing --cov-fail-under=50
+addopts = --cov=src --cov=common --cov=utils --cov-report=term-missing --cov-fail-under=20
 env =
     DEV_ENV = dev
     OPENAI_API_KEY=test_api_key