Fix: Eval hash mismatch due to parameter truncation in DB storage

pyrit/identifiers/component_identifier.py

@@ -113,6 +113,7 @@ class ComponentIdentifier:
     KEY_CLASS_NAME: ClassVar[str] = "class_name"
     KEY_CLASS_MODULE: ClassVar[str] = "class_module"
     KEY_HASH: ClassVar[str] = "hash"
+    KEY_EVAL_HASH: ClassVar[str] = "eval_hash"
     KEY_PYRIT_VERSION: ClassVar[str] = "pyrit_version"
     KEY_CHILDREN: ClassVar[str] = "children"
     LEGACY_KEY_TYPE: ClassVar[str] = "__type__"
@@ -130,6 +131,10 @@ class ComponentIdentifier:
     hash: str = field(init=False, compare=False)
     #: Version tag for storage. Not included in hash.
     pyrit_version: str = field(default_factory=lambda: pyrit.__version__, compare=False)
+    #: Evaluation hash preserved from DB round-trip. Computed before truncation and
+    #: stored alongside the identity so that EvaluationIdentifier can use it directly
+    #: instead of recomputing from potentially truncated params.
+    stored_eval_hash: Optional[str] = field(default=None, init=False, compare=False)
 
     def __post_init__(self) -> None:
         """Compute the content-addressed hash at creation time."""
@@ -231,7 +236,7 @@ def normalize(cls, value: Union[ComponentIdentifier, dict[str, Any]]) -> Compone
             return cls.from_dict(value)
         raise TypeError(f"Expected ComponentIdentifier or dict, got {type(value).__name__}")
 
-    def to_dict(self, *, max_value_length: Optional[int] = None) -> dict[str, Any]:
+    def to_dict(self, *, max_value_length: Optional[int] = None, eval_hash: Optional[str] = None) -> dict[str, Any]:
         """
         Serialize to a JSON-compatible dictionary for DB/JSONL storage.
 
@@ -246,6 +251,10 @@ def to_dict(self, *, max_value_length: Optional[int] = None) -> dict[str, Any]:
                 DB storage where column sizes may be limited. The truncation applies
                 only to param values, not to structural keys like class_name or hash.
                 The limit is propagated to children. Defaults to None (no truncation).
+            eval_hash (Optional[str]): If provided, the evaluation hash is included in
+                the serialized dict. This should be computed before truncation so that
+                it can be recovered via ``from_dict()`` even when param values are
+                truncated. Defaults to None (no eval_hash stored).
 
         Returns:
             Dict[str, Any]: JSON-serializable dictionary suitable for database storage
@@ -258,6 +267,11 @@ def to_dict(self, *, max_value_length: Optional[int] = None) -> dict[str, Any]:
             self.KEY_PYRIT_VERSION: self.pyrit_version,
         }
 
+        # Include eval_hash if explicitly provided or if preserved from a prior round-trip
+        effective_eval_hash = eval_hash if eval_hash is not None else self.stored_eval_hash
+        if effective_eval_hash is not None:
+            result[self.KEY_EVAL_HASH] = effective_eval_hash
+
         for key, value in self.params.items():
             result[key] = self._truncate_value(value=value, max_length=max_value_length)
 
@@ -324,6 +338,7 @@ def from_dict(cls, data: dict[str, Any]) -> ComponentIdentifier:
         class_module = data.pop(cls.KEY_CLASS_MODULE, None) or data.pop(cls.LEGACY_KEY_MODULE, None) or "unknown"
 
         stored_hash = data.pop(cls.KEY_HASH, None)
+        stored_eval_hash = data.pop(cls.KEY_EVAL_HASH, None)
         pyrit_version = data.pop(cls.KEY_PYRIT_VERSION, pyrit.__version__)
 
         # Reconstruct children
@@ -346,6 +361,11 @@ def from_dict(cls, data: dict[str, Any]) -> ComponentIdentifier:
         if stored_hash:
             object.__setattr__(identifier, "hash", stored_hash)
 
+        # Preserve stored eval_hash if available — computed before truncation
+        # so that EvaluationIdentifier can use it directly.
+        if stored_eval_hash:
+            object.__setattr__(identifier, "stored_eval_hash", stored_eval_hash)
+
         return identifier
 
     def get_child(self, key: str) -> Optional[ComponentIdentifier]:

pyrit/identifiers/evaluation_identifier.py

@@ -170,12 +170,22 @@ class EvaluationIdentifier(ABC):
     CHILD_EVAL_RULES: ClassVar[dict[str, ChildEvalRule]]
 
     def __init__(self, identifier: ComponentIdentifier) -> None:
-        """Wrap a ComponentIdentifier and eagerly compute its eval hash."""
+        """
+        Wrap a ComponentIdentifier and resolve its eval hash.
+
+        If the identifier carries a ``stored_eval_hash`` (preserved from a prior
+        DB round-trip), that value is used directly. Otherwise the eval hash is
+        computed from the identifier's params and children using the subclass's
+        ``CHILD_EVAL_RULES``.
+        """
         self._identifier = identifier
-        self._eval_hash = compute_eval_hash(
-            identifier,
-            child_eval_rules=self.CHILD_EVAL_RULES,
-        )
+        if identifier.stored_eval_hash is not None:
+            self._eval_hash = identifier.stored_eval_hash
+        else:
+            self._eval_hash = compute_eval_hash(
+                identifier,
+                child_eval_rules=self.CHILD_EVAL_RULES,
+            )
 
     @property
     def identifier(self) -> ComponentIdentifier:

pyrit/memory/memory_models.py

@@ -33,6 +33,10 @@
 import pyrit
 from pyrit.common.utils import to_sha256
 from pyrit.identifiers.component_identifier import ComponentIdentifier
+from pyrit.identifiers.evaluation_identifier import (
+    AtomicAttackEvaluationIdentifier,
+    ScorerEvaluationIdentifier,
+)
 from pyrit.models import (
     AttackOutcome,
     AttackResult,
@@ -51,6 +55,8 @@
     SeedType,
 )
 
+logger = logging.getLogger(__name__)
+
 # Default pyrit_version for database records created before version tracking was added
 LEGACY_PYRIT_VERSION = "<0.10.0"
 
@@ -398,7 +404,12 @@ def __init__(self, *, entry: Score):
         self.score_metadata = entry.score_metadata
         # Normalize to ComponentIdentifier (handles dict with deprecation warning) then convert to dict for JSON storage
         normalized_scorer = ComponentIdentifier.normalize(entry.scorer_class_identifier)
-        self.scorer_class_identifier = normalized_scorer.to_dict(max_value_length=MAX_IDENTIFIER_VALUE_LENGTH)
+        # Compute eval_hash before truncation so it survives the DB round-trip
+        scorer_eval_hash = self._get_scorer_eval_hash(normalized_scorer)
+        self.scorer_class_identifier = normalized_scorer.to_dict(
+            max_value_length=MAX_IDENTIFIER_VALUE_LENGTH,
+            eval_hash=scorer_eval_hash,
+        )
         self.prompt_request_response_id = entry.message_piece_id if entry.message_piece_id else None
         self.timestamp = entry.timestamp
         # Store in both columns for backward compatibility
@@ -407,6 +418,23 @@ def __init__(self, *, entry: Score):
         self.objective = entry.objective
         self.pyrit_version = pyrit.__version__
 
+    @staticmethod
+    def _get_scorer_eval_hash(scorer_identifier: ComponentIdentifier) -> Optional[str]:
+        """
+        Compute scorer eval_hash from the given identifier.
+
+        Returns:
+            Optional[str]: The eval_hash, or None if computation fails.
+        """
+        try:
+            return ScorerEvaluationIdentifier(scorer_identifier).eval_hash
+        except Exception:
+            logger.warning(
+                f"Failed to compute eval_hash for scorer {scorer_identifier.class_name}; eval_hash will not be stored.",
+                exc_info=True,
+            )
+            return None
+
     def get_score(self) -> Score:
         """
         Convert this database entry back into a Score object.
@@ -770,8 +798,13 @@ def __init__(self, *, entry: AttackResult):
         self.attack_identifier = (
             _attack_strategy_id.to_dict(max_value_length=MAX_IDENTIFIER_VALUE_LENGTH) if _attack_strategy_id else {}
         )
+        # Compute eval_hash before truncation so it survives the DB round-trip
+        attack_eval_hash = self._get_attack_eval_hash(entry.atomic_attack_identifier)
         self.atomic_attack_identifier = (
-            entry.atomic_attack_identifier.to_dict(max_value_length=MAX_IDENTIFIER_VALUE_LENGTH)
+            entry.atomic_attack_identifier.to_dict(
+                max_value_length=MAX_IDENTIFIER_VALUE_LENGTH,
+                eval_hash=attack_eval_hash,
+            )
             if entry.atomic_attack_identifier
             else None
         )
@@ -799,6 +832,26 @@ def __init__(self, *, entry: AttackResult):
         self.timestamp = datetime.now(tz=timezone.utc)
         self.pyrit_version = pyrit.__version__
 
+    @staticmethod
+    def _get_attack_eval_hash(attack_identifier: Optional[ComponentIdentifier]) -> Optional[str]:
+        """
+        Compute attack eval_hash from the given identifier.
+
+        Returns:
+            Optional[str]: The eval_hash, or None if identifier is None or computation fails.
+        """
+        if attack_identifier is None:
+            return None
+
+        try:
+            return AtomicAttackEvaluationIdentifier(attack_identifier).eval_hash
+        except Exception:
+            logger.warning(
+                f"Failed to compute eval_hash for attack {attack_identifier.class_name}; eval_hash will not be stored.",
+                exc_info=True,
+            )
+            return None
+
     @staticmethod
     def _get_id_as_uuid(obj: Any) -> Optional[uuid.UUID]:
         """
@@ -974,9 +1027,17 @@ def __init__(self, *, entry: ScenarioResult):
         self.objective_target_identifier = entry.objective_target_identifier.to_dict(
             max_value_length=MAX_IDENTIFIER_VALUE_LENGTH
         )
-        # Convert ComponentIdentifier to dict for JSON storage
+        # Compute eval_hash before truncation so it survives the DB round-trip, then include
+        # it in the serialized dict so it survives the DB round-trip.
+        scorer_eval_hash = None
+        if entry.objective_scorer_identifier:
+            scorer_eval_hash = ScorerEvaluationIdentifier(entry.objective_scorer_identifier).eval_hash
+
         self.objective_scorer_identifier = (
-            entry.objective_scorer_identifier.to_dict(max_value_length=MAX_IDENTIFIER_VALUE_LENGTH)
+            entry.objective_scorer_identifier.to_dict(
+                max_value_length=MAX_IDENTIFIER_VALUE_LENGTH,
+                eval_hash=scorer_eval_hash,
+            )
             if entry.objective_scorer_identifier
             else None
         )

pyrit/scenario/core/atomic_attack.py

@@ -13,12 +13,14 @@
 have a common interface for scenarios.
 """
 
+import contextlib
 import logging
 from typing import TYPE_CHECKING, Any, Optional
 
 from pyrit.executor.attack import AttackExecutor, AttackStrategy
 from pyrit.executor.attack.core.attack_executor import AttackExecutorResult
 from pyrit.identifiers import build_atomic_attack_identifier
+from pyrit.identifiers.evaluation_identifier import AtomicAttackEvaluationIdentifier
 from pyrit.memory import CentralMemory
 from pyrit.memory.memory_models import MAX_IDENTIFIER_VALUE_LENGTH
 from pyrit.models import AttackResult, SeedAttackGroup
@@ -251,13 +253,19 @@ def _enrich_atomic_attack_identifiers(self, *, results: AttackExecutorResult[Att
                     seed_group=self._seed_groups[idx],
                 )
 
-                # Persist the enriched identifier back to the database
+                # Persist the enriched identifier back to the database.
+                # Compute eval_hash before truncation so it survives the DB round-trip.
+                attack_eval_hash = None
+                with contextlib.suppress(Exception):
+                    attack_eval_hash = AtomicAttackEvaluationIdentifier(result.atomic_attack_identifier).eval_hash
+
                 if result.attack_result_id:
                     memory.update_attack_result_by_id(
                         attack_result_id=result.attack_result_id,
                         update_fields={
                             "atomic_attack_identifier": result.atomic_attack_identifier.to_dict(
-                                max_value_length=MAX_IDENTIFIER_VALUE_LENGTH
+                                max_value_length=MAX_IDENTIFIER_VALUE_LENGTH,
+                                eval_hash=attack_eval_hash,
                             ),
                         },
                     )

pyrit/score/scorer.py

@@ -70,6 +70,25 @@ def __init__(self, *, validator: ScorerPromptValidator):
         """
         self._validator = validator
 
+    def get_identifier(self) -> ComponentIdentifier:
+        """
+        Get the scorer's identifier with eval_hash always attached.
+
+        Overrides the base ``Identifiable.get_identifier()`` so that
+        ``to_dict()`` always emits the ``eval_hash`` key. This lets consumers
+        see at a glance whether the scorer matches a registry entry.
+
+        Returns:
+            ComponentIdentifier: The identity with ``stored_eval_hash`` set.
+        """
+        identifier = super().get_identifier()
+        if identifier.stored_eval_hash is None:
+            from pyrit.identifiers.evaluation_identifier import ScorerEvaluationIdentifier
+
+            eval_hash = ScorerEvaluationIdentifier(identifier).eval_hash
+            object.__setattr__(identifier, "stored_eval_hash", eval_hash)
+        return identifier
+
     def get_eval_hash(self) -> str:
         """
         Compute a behavioral equivalence hash for evaluation grouping.
@@ -81,10 +100,7 @@ def get_eval_hash(self) -> str:
         Returns:
             str: A hex-encoded SHA256 hash suitable for eval registry keying.
         """
-        # Deferred import to avoid circular dependency (evaluation_identifier → identifiers → …)
-        from pyrit.identifiers.evaluation_identifier import ScorerEvaluationIdentifier
-
-        return ScorerEvaluationIdentifier(self.get_identifier()).eval_hash
+        return self.get_identifier().stored_eval_hash
 
     @property
     def scorer_type(self) -> ScoreType:

tests/unit/identifiers/test_component_identifier.py

@@ -544,6 +544,100 @@ def test_roundtrip_with_list_children(self):
         assert isinstance(recon_converters, list)
         assert len(recon_converters) == 2
 
+    def test_roundtrip_preserves_eval_hash(self):
+        """Test that eval_hash is preserved through to_dict -> from_dict round-trip."""
+        original = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            params={"system_prompt": "Score the response"},
+        )
+        expected_eval_hash = "abc123" * 10 + "abcd"  # 64 chars
+        d = original.to_dict(eval_hash=expected_eval_hash)
+        assert d["eval_hash"] == expected_eval_hash
+
+        reconstructed = ComponentIdentifier.from_dict(d)
+        assert reconstructed.stored_eval_hash == expected_eval_hash
+
+    def test_roundtrip_eval_hash_survives_truncation(self):
+        """Regression test: eval_hash computed before truncation is preserved after round-trip.
+
+        This is the core bug fix — long params get truncated in to_dict(), which would
+        cause eval_hash recomputation to produce a wrong hash. By storing eval_hash in
+        the dict, it survives truncation.
+        """
+        long_prompt = "You are a scorer that evaluates responses. " * 20  # >80 chars
+        original = ComponentIdentifier(
+            class_name="SelfAskTrueFalseScorer",
+            class_module="pyrit.score",
+            params={"system_prompt_template": long_prompt},
+        )
+        eval_hash_before_truncation = "correct_eval_hash_" + "0" * 46  # 64 chars
+
+        # Serialize with truncation AND eval_hash (simulates DB storage)
+        truncated_dict = original.to_dict(max_value_length=80, eval_hash=eval_hash_before_truncation)
+        # Params are truncated
+        assert truncated_dict["system_prompt_template"].endswith("...")
+        # But eval_hash is preserved
+        assert truncated_dict["eval_hash"] == eval_hash_before_truncation
+
+        # Deserialize
+        reconstructed = ComponentIdentifier.from_dict(truncated_dict)
+        # eval_hash is available on the reconstructed identifier
+        assert reconstructed.stored_eval_hash == eval_hash_before_truncation
+        # And it's NOT in params (from_dict pops it as a reserved key)
+        assert "eval_hash" not in reconstructed.params
+
+    def test_roundtrip_no_eval_hash_when_not_provided(self):
+        """Test that stored_eval_hash is None when not included in serialization."""
+        original = ComponentIdentifier(
+            class_name="Test",
+            class_module="mod",
+            params={"key": "value"},
+        )
+        d = original.to_dict()
+        assert "eval_hash" not in d
+
+        reconstructed = ComponentIdentifier.from_dict(d)
+        assert reconstructed.stored_eval_hash is None
+
+    def test_to_dict_includes_stored_eval_hash_from_prior_roundtrip(self):
+        """Test that to_dict re-emits stored_eval_hash from a prior round-trip."""
+        eval_hash = "deadbeef" * 8  # 64 chars
+        original = ComponentIdentifier(
+            class_name="Test",
+            class_module="mod",
+        )
+        # Simulate a prior round-trip that stored an eval_hash
+        d1 = original.to_dict(eval_hash=eval_hash)
+        reconstructed = ComponentIdentifier.from_dict(d1)
+
+        # Re-serialize without explicitly passing eval_hash — stored one should be emitted
+        d2 = reconstructed.to_dict()
+        assert d2["eval_hash"] == eval_hash
+
+    def test_double_roundtrip_preserves_eval_hash_and_identity_hash(self):
+        """Test that both eval_hash and identity hash survive retrieve → re-store → retrieve."""
+        long_prompt = "Score the response carefully. " * 20
+        original = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            params={"system_prompt": long_prompt},
+        )
+        original_hash = original.hash
+        eval_hash = "eval_" + "a1b2c3d4" * 7 + "a1b2c3"  # 64 chars
+
+        # First round-trip: store with truncation
+        d1 = original.to_dict(max_value_length=80, eval_hash=eval_hash)
+        r1 = ComponentIdentifier.from_dict(d1)
+        assert r1.hash == original_hash
+        assert r1.stored_eval_hash == eval_hash
+
+        # Second round-trip: re-store (simulating retrieve → use → re-store)
+        d2 = r1.to_dict(max_value_length=80)
+        r2 = ComponentIdentifier.from_dict(d2)
+        assert r2.hash == original_hash
+        assert r2.stored_eval_hash == eval_hash
+
 
 class TestComponentIdentifierNormalize:
     """Tests for normalize class method."""