Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/agentpool/models/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,7 @@ class AgentsManifest(Schema):
"""

model_config = ConfigDict(
extra="allow",
json_schema_extra={
"x-icon": "octicon:file-code-16",
"x-doc-title": "Manifest Overview",
Expand Down Expand Up @@ -777,6 +778,31 @@ def get_output_type(self, agent_name: str) -> type[Any] | None:
return response_def.response_schema.get_schema()
return agent_config.output_type.response_schema.get_schema()

@model_validator(mode="after")
def validate_extra_fields(self) -> Self:
"""Validate and warn about unknown extra fields.

Allowed prefixes:
- `.` (dot): YAML anchors
- `_` (underscore): Internal metadata
- `x-` (x-prefix): Custom extensions

Unknown fields trigger a WARNING but do not raise ValidationError.
"""
if hasattr(self, "model_extra") and self.model_extra:
for key in self.model_extra:
# Check if key starts with allowed prefixes
if key.startswith((".", "_", "x-")):
continue # Silently allow these

# Warn about unknown fields
logger.warning(
"Unknown field '%s' in manifest. This field will be IGNORED.",
key,
stacklevel=2,
)
return self


if __name__ == "__main__":
from llmling_models_config import InputModelConfig
Expand Down
114 changes: 114 additions & 0 deletions tests/manifest/test_metadata_fields.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
"""Tests for manifest metadata fields (YAML anchors and extensions)."""

from __future__ import annotations

from llmling_models_config import StringModelConfig
import yamling

from agentpool import AgentsManifest
from agentpool.models.agents import NativeAgentConfig


# Valid config with allowed metadata fields
MANIFEST_WITH_ALLOWED_METADATA = """\
agents:
test_agent:
type: native
model: openai:gpt-4o
system_prompt: "You are a test agent"

.anchor: &default_settings
timeout: 30
retries: 3

_meta:
version: "1.0.0"
author: "Test User"

x-custom:
environment: "production"
feature_flags:
- feature_a
- feature_b
"""

# Valid config with unknown field (typo)
MANIFEST_WITH_UNKNOWN_FIELD = """\
agents:
test_agent:
type: native
model: openai:gpt-4o
system_prompt: "You are a test agent"

random_field: "this is a typo/unknown field"
"""

# Valid config with both allowed and unknown fields
MANIFEST_WITH_MIXED_FIELDS = """\
agents:
test_agent:
type: native
model: openai:gpt-4o
system_prompt: "You are a test agent"

_meta:
version: "1.0.0"

random_field: "should trigger warning"
"""


def test_allowed_metadata_fields_succeed():
"""Test that metadata fields starting with ., _, x- are allowed.

RED PHASE: This test will FAIL because currently extra fields
are forbidden. After implementation, this test will PASS.
"""
config = yamling.load_yaml(MANIFEST_WITH_ALLOWED_METADATA)
manifest = AgentsManifest.model_validate(config)

# Verify that manifest loaded successfully
assert "test_agent" in manifest.agents
agent = manifest.agents["test_agent"]
assert isinstance(agent, NativeAgentConfig)
assert isinstance(agent.model, StringModelConfig)
assert agent.model.identifier == "openai:gpt-4o"


def test_unknown_field_generates_warning():
"""Test that unknown fields generate a warning but don't raise ValidationError.

RED PHASE: This test will FAIL because currently unknown fields
raise ValidationError. After implementation, this test will PASS.
"""
config = yamling.load_yaml(MANIFEST_WITH_UNKNOWN_FIELD)

# After implementation, this should NOT raise ValidationError
# It should log a warning instead
manifest = AgentsManifest.model_validate(config)

# Verify agents loaded correctly
assert "test_agent" in manifest.agents
agent = manifest.agents["test_agent"]
assert isinstance(agent, NativeAgentConfig)
assert isinstance(agent.model, StringModelConfig)
assert agent.model.identifier == "openai:gpt-4o"


def test_mixed_allowed_and_unknown_fields():
"""Test manifest with both allowed and unknown fields.

RED PHASE: This test will FAIL because currently unknown fields
raise ValidationError. After implementation, this test will PASS.
"""
config = yamling.load_yaml(MANIFEST_WITH_MIXED_FIELDS)

# After implementation, this should succeed with warning for 'random_field'
manifest = AgentsManifest.model_validate(config)

# Verify allowed metadata fields are accessible
assert "test_agent" in manifest.agents
agent = manifest.agents["test_agent"]
assert isinstance(agent, NativeAgentConfig)
assert isinstance(agent.model, StringModelConfig)
assert agent.model.identifier == "openai:gpt-4o"
15 changes: 10 additions & 5 deletions tests/manifest/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,16 @@
"""

INVALID_RESPONSE_CONFIG = """\
responses: {}
agent:
name: Test Agent
model: test
output_type: NonExistentResponse
responses:
InvalidResponse:
type: object
agents:
test_agent:
type: native
model: "openai:gpt-4o"
system_prompt: "test"
output_type: NonExistentResponse
"""


Expand Down
Loading