Eval/redux

Cargo.toml

@@ -78,6 +78,7 @@ pyo3 = { version = "0.*", features = ["abi3-py310", "chrono", "anyhow", "extensi
 rand = "0.9.*"
 rayon = "1.*"
 redis = { version = "0.*", features = ["tokio-comp"] }
+regex = "1.*"
 reqwest = { version = "0.12.*", features = ["json", "multipart", "rustls-tls", "rustls-tls-native-roots", "blocking" ], default-features = false }
 rusty-logging="0.*"
 semver = "1.*"

crates/scouter_client/src/drifter/llm.rs

@@ -1,7 +1,7 @@
 use pyo3::prelude::*;
 use scouter_drift::{error::DriftError, LLMEvaluator};
 use scouter_state::app_state;
-use scouter_types::llm::{LLMDriftConfig, LLMDriftMetric, LLMDriftProfile};
+use scouter_types::genai::{GenAIDriftConfig, LLMDriftMetric, GenAIDriftProfile};
 use scouter_types::{LLMMetricRecord, LLMRecord};
 use std::sync::Arc;
 
@@ -24,17 +24,17 @@ impl ClientLLMDrifter {
 
     pub fn create_drift_profile(
         &mut self,
-        config: LLMDriftConfig,
+        config: GenAIDriftConfig,
         metrics: Vec<LLMDriftMetric>,
         workflow: Option<Bound<'_, PyAny>>,
-    ) -> Result<LLMDriftProfile, DriftError> {
-        let profile = LLMDriftProfile::new(config, metrics, workflow)?;
+    ) -> Result<GenAIDriftProfile, DriftError> {
+        let profile = GenAIDriftProfile::new(config, metrics, workflow)?;
         Ok(profile)
     }
 
     pub async fn compute_drift_single(
         record: &LLMRecord,
-        profile: &LLMDriftProfile,
+        profile: &GenAIDriftProfile,
     ) -> Result<Vec<LLMMetricRecord>, DriftError> {
         let (metrics, _score, _workflow_duration) =
             LLMEvaluator::process_drift_record(record, profile).await?;
@@ -44,7 +44,7 @@ impl ClientLLMDrifter {
     pub fn compute_drift(
         &mut self,
         data: Vec<LLMRecord>,
-        profile: &LLMDriftProfile,
+        profile: &GenAIDriftProfile,
     ) -> Result<Vec<LLMMetricRecord>, DriftError> {
         let results = self.runtime.block_on(async move {
             let mut results = Vec::new();

crates/scouter_client/src/drifter/scouter.rs

@@ -7,12 +7,12 @@ use pyo3::types::PyList;
 use pyo3::IntoPyObjectExt;
 use scouter_drift::error::DriftError;
 use scouter_drift::spc::SpcDriftMap;
-use scouter_types::llm::{LLMDriftMap, LLMDriftMetric};
+use scouter_types::genai::{GenAIDriftMap, LLMDriftMetric};
 use scouter_types::spc::SpcDriftProfile;
 use scouter_types::LLMRecord;
 use scouter_types::{
     custom::{CustomDriftProfile, CustomMetric, CustomMetricDriftConfig},
-    llm::{LLMDriftConfig, LLMDriftProfile},
+    genai::{GenAIDriftConfig, GenAIDriftProfile},
     psi::{PsiDriftConfig, PsiDriftMap, PsiDriftProfile},
     spc::SpcDriftConfig,
     DataType, DriftProfile, DriftType,
@@ -24,13 +24,13 @@ use std::sync::RwLock;
 pub enum DriftMap {
     Spc(SpcDriftMap),
     Psi(PsiDriftMap),
-    LLM(LLMDriftMap),
+    LLM(GenAIDriftMap),
 }
 
 pub enum DriftConfig {
     Spc(Arc<RwLock<SpcDriftConfig>>),
     Psi(Arc<RwLock<PsiDriftConfig>>),
-    LLM(LLMDriftConfig),
+    LLM(GenAIDriftConfig),
     Custom(CustomMetricDriftConfig),
 }
 
@@ -56,7 +56,7 @@ impl DriftConfig {
         }
     }
 
-    pub fn llm_config(&self) -> Result<LLMDriftConfig, DriftError> {
+    pub fn llm_config(&self) -> Result<GenAIDriftConfig, DriftError> {
         match self {
             DriftConfig::LLM(cfg) => Ok(cfg.clone()),
             _ => Err(DriftError::InvalidConfigError),
@@ -88,7 +88,7 @@ impl Drifter {
             DriftType::Spc => Ok(Drifter::Spc(SpcDrifter::new())),
             DriftType::Psi => Ok(Drifter::Psi(PsiDrifter::new())),
             DriftType::Custom => Ok(Drifter::Custom(CustomDrifter::new())),
-            DriftType::LLM => Ok(Drifter::LLM(ClientLLMDrifter::new())),
+            DriftType::GenAI => Ok(Drifter::LLM(ClientLLMDrifter::new())),
         }
     }
 
@@ -136,7 +136,7 @@ impl Drifter {
                 };
                 let profile =
                     drifter.create_drift_profile(config.llm_config()?, metrics, workflow)?;
-                Ok(DriftProfile::LLM(profile))
+                Ok(DriftProfile::GenAI(profile))
             }
         }
     }
@@ -176,7 +176,7 @@ impl Drifter {
                 };
                 let records = drifter.compute_drift(data, profile.get_llm_profile()?)?;
 
-                Ok(DriftMap::LLM(LLMDriftMap { records }))
+                Ok(DriftMap::LLM(GenAIDriftMap { records }))
             }
         }
     }
@@ -228,8 +228,8 @@ impl PyDrifter {
                     let config = obj.extract::<CustomMetricDriftConfig>()?;
                     DriftConfig::Custom(config)
                 }
-                DriftType::LLM => {
-                    let config = obj.extract::<LLMDriftConfig>()?;
+                DriftType::GenAI => {
+                    let config = obj.extract::<GenAIDriftConfig>()?;
                     DriftConfig::LLM(config)
                 }
             };
@@ -265,7 +265,7 @@ impl PyDrifter {
             DriftProfile::Spc(profile) => Ok(profile.into_bound_py_any(py)?),
             DriftProfile::Psi(profile) => Ok(profile.into_bound_py_any(py)?),
             DriftProfile::Custom(profile) => Ok(profile.into_bound_py_any(py)?),
-            DriftProfile::LLM(profile) => Ok(profile.into_bound_py_any(py)?),
+            DriftProfile::GenAI(profile) => Ok(profile.into_bound_py_any(py)?),
         }
     }
 
@@ -275,11 +275,11 @@ impl PyDrifter {
     pub fn create_llm_drift_profile<'py>(
         &mut self,
         py: Python<'py>,
-        config: LLMDriftConfig,
+        config: GenAIDriftConfig,
         metrics: Vec<LLMDriftMetric>,
         workflow: Option<Bound<'py, PyAny>>,
     ) -> Result<Bound<'py, PyAny>, DriftError> {
-        let profile = LLMDriftProfile::new(config, metrics, workflow)?;
+        let profile = GenAIDriftProfile::new(config, metrics, workflow)?;
         Ok(profile.into_bound_py_any(py)?)
     }
 
@@ -309,9 +309,9 @@ impl PyDrifter {
                 let profile = drift_profile.extract::<CustomDriftProfile>()?;
                 DriftProfile::Custom(profile)
             }
-            DriftType::LLM => {
-                let profile = drift_profile.extract::<LLMDriftProfile>()?;
-                DriftProfile::LLM(profile)
+            DriftType::GenAI => {
+                let profile = drift_profile.extract::<GenAIDriftProfile>()?;
+                DriftProfile::GenAI(profile)
             }
         };
 
@@ -322,7 +322,7 @@ impl PyDrifter {
         let data_type = match data_type {
             Some(data_type) => data_type,
             None => {
-                if drift_type == DriftType::LLM {
+                if drift_type == DriftType::GenAI {
                     // For LLM, we will handle it separately in the create_llm_drift_profile method
                     &DataType::LLM
                 } else {
@@ -355,12 +355,12 @@ impl PyDrifter {
     pub fn create_llm_drift_profile_with_runtime<'py>(
         &mut self,
         py: Python<'py>,
-        config: LLMDriftConfig,
+        config: GenAIDriftConfig,
         metrics: Vec<LLMDriftMetric>,
         workflow: Option<Bound<'py, PyAny>>,
         runtime: Arc<tokio::runtime::Runtime>,
     ) -> Result<Bound<'py, PyAny>, DriftError> {
-        let profile = LLMDriftProfile::new_with_runtime(config, metrics, workflow, runtime)?;
+        let profile = GenAIDriftProfile::new_with_runtime(config, metrics, workflow, runtime)?;
         Ok(profile.into_bound_py_any(py)?)
     }
 }

crates/scouter_client/src/http/client.rs

@@ -288,7 +288,7 @@ impl PyScouterClient {
             DriftType::Custom => {
                 PyScouterClient::get_custom_binned_drift(py, &self.client.client, drift_request)
             }
-            DriftType::LLM => {
+            DriftType::GenAI => {
                 PyScouterClient::get_llm_metric_binned_drift(py, &self.client.client, drift_request)
             }
         }

crates/scouter_client/src/lib.rs

@@ -17,7 +17,7 @@ pub use scouter_types::{
     },
     eval::LLMEvalMetric,
     llm::{
-        LLMAlertConfig, LLMDriftConfig, LLMDriftMap, LLMDriftMetric, LLMDriftProfile,
+        GenAIAlertConfig, GenAIDriftConfig, GenAIDriftMap, GenAIDriftProfile, LLMDriftMetric,
         LLMMetricAlertCondition, PaginationCursor, PaginationResponse,
     },
     psi::{
@@ -72,6 +72,6 @@ pub use scouter_types::error::{ContractError, ProfileError, RecordError, TypeErr
 
 pub use scouter_evaluate::{
     error::EvaluationError,
-    llm::{async_evaluate_llm, evaluate_llm, workflow_from_eval_metrics},
+    evaluate::{async_evaluate_llm, evaluate_llm, workflow_from_eval_metrics},
     types::{EvaluationConfig, LLMEvalRecord, LLMEvalResults, LLMEvalTaskResult},
 };