Bug 1997631 - Add telemetry for database loads and migrations to Nimbus SDK

There are now two new events, `nimbus_events.database_load`, which is
emitted every time we attempt to load the database (i.e., once per
session), and `nimbus_events.database_migration`, which is emitted every
time we attempt a database migration. These events should give us
insight into how often we are experiencing database corruption leading
to enrollment state being dropped.

Author: freshstrangemusic

components/nimbus/android/src/main/java/org/mozilla/experiments/nimbus/Nimbus.kt

@@ -32,6 +32,8 @@ import org.mozilla.experiments.nimbus.GleanMetrics.NimbusHealth
 import org.mozilla.experiments.nimbus.GleanMetrics.Pings
 import org.mozilla.experiments.nimbus.internal.AppContext
 import org.mozilla.experiments.nimbus.internal.AvailableExperiment
+import org.mozilla.experiments.nimbus.internal.DatabaseLoadExtraDef
+import org.mozilla.experiments.nimbus.internal.DatabaseMigrationExtraDef
 import org.mozilla.experiments.nimbus.internal.EnrolledExperiment
 import org.mozilla.experiments.nimbus.internal.EnrollmentChangeEvent
 import org.mozilla.experiments.nimbus.internal.EnrollmentChangeEventType
@@ -83,6 +85,27 @@ open class Nimbus(
     private val logger = delegate.logger
 
     private val metricsHandler = object : MetricsHandler {
+        override fun recordDatabaseLoad(event: DatabaseLoadExtraDef) {
+            NimbusEvents.databaseLoad.record(
+                NimbusEvents.DatabaseLoadExtra(
+                    corrupt = event.corrupt,
+                    initialversion = event.initialVersion,
+                    error = event.error,
+                    migratedVersion = event.migratedVersion,
+                    migrationError = event.migrationError,
+                )
+            )
+        }
+        override fun recordDatabaseMigration(event: DatabaseMigrationExtraDef) {
+            NimbusEvents.databaseMigrated.record(
+                NimbusEvents.DatabaseMigratedExtra(
+                    reason = event.reason,
+                    fromVersion = event.fromVersion,
+                    toVersion = event.toVersion,
+                    error = event.error,
+                )
+            )
+        }
         override fun recordEnrollmentStatuses(enrollmentStatusExtras: List<EnrollmentStatusExtraDef>) {
             for (extra in enrollmentStatusExtras) {
                 NimbusEvents.enrollmentStatus.record(

components/nimbus/metrics.yaml

@@ -167,6 +167,89 @@ nimbus_events:
     expires: never
     # Disabled by default. This needs a server-knobs rollout to ensure the volume is not overwhelming.
     disabled: true
+
+  database_load:
+    type: event
+    description: >
+      An event recorded when the Nimbus database is loaded.
+
+    extra_keys:
+      corrupt:
+        type: boolean
+        description: >
+          Whether or not the database was corrupted when we tried to load it.
+
+          This field may be empty, e.g., if the database could not be loaded.
+
+      initial_version:
+        type: quantity
+        description: >
+          The version of the database on disk at load time.
+
+          This field may be empty, e.g., if we cannot create a database.
+
+      error:
+        type: string
+        description: >
+          If an error occured during the initial load of the database this field
+          will contain the relevant error code.
+
+      migrated_version:
+        type: quantity
+        description: >
+          The version of the database that was migrated to.
+
+          This field will be empty if no migration(s) occurred.
+
+      migration_error:
+        type: string
+        description: >
+          If an error occurred during the initial load of the database this
+          field will contain the relevant error code.
+
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997631
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1997631
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - beth@mozilla.com
+      - project-nimbus@mozilla.com
+    expires: never
+
+    database_migration:
+      type: events
+      description: >
+        An event recorded when a database migration occurs.
+
+      extra_keys:
+        reason:
+          type: string
+          description: >
+            The reason the migration occurred.
+
+            This is one of:
+
+            - "upgrade"
+            - "invalid_version"
+
+        from_version:
+          type: quantity
+          description: >
+            The original version of the database. A value of 0 indicates that
+            the db_version was not set in the meta store.
+
+        to_version:
+          type: quantity
+          description: >
+            The target version of the migration that was performed.
+
+        error:
+          type: string
+          description: >
+            If an error occurred during the migration, the relevant error code.
+
   exposure:
     type: event
     description: >

components/nimbus/src/error.rs

@@ -8,6 +8,7 @@
 //! TODO: Implement proper error handling, this would include defining the error enum,
 //! impl std::error::Error using `thiserror` and ensuring all errors are handled appropriately
 
+use std::borrow::Cow;
 use std::num::{ParseIntError, TryFromIntError};
 
 // reexport logging helpers.
@@ -71,12 +72,12 @@ pub enum NimbusError {
     #[error("TryInto error: {0}")]
     TryFromSliceError(#[from] std::array::TryFromSliceError),
 
-    #[error("Error parsing URL: {0}")]
-    UrlParsingError(#[from] url::ParseError),
-
     #[error("UniFFI callback error: {0}")]
     UniFFICallbackError(#[from] uniffi::UnexpectedUniFFICallbackError),
 
+    #[error("Error parsing URL: {0}")]
+    UrlParsingError(#[from] url::ParseError),
+
     #[error("UUID parsing error: {0}")]
     UuidError(#[from] uuid::Error),
 
@@ -92,14 +93,14 @@ pub enum NimbusError {
     #[error("Error with Remote Settings client: {0}")]
     ClientError(#[from] remote_settings::RemoteSettingsError),
 
-    #[cfg(feature = "stateful")]
-    #[error("Rkv error: {0}")]
-    RkvError(#[from] rkv::StoreError),
-
     #[cfg(feature = "stateful")]
     #[error("Regex error: {0}")]
     RegexError(#[from] regex::Error),
 
+    #[cfg(feature = "stateful")]
+    #[error("Rkv error: {0}")]
+    RkvError(#[from] rkv::StoreError),
+
     // Cirrus-only errors.
     #[cfg(not(feature = "stateful"))]
     #[error("Error in Cirrus: {0}")]
@@ -162,3 +163,83 @@ impl From<VersionParsingError> for NimbusError {
 }
 
 pub type Result<T, E = NimbusError> = std::result::Result<T, E>;
+
+/// An Error extension trait that allows simplified error codes to be submitted
+/// in telemetry.
+pub trait ErrorCode: std::error::Error {
+    /// Return the error code for the given error.
+    fn error_code(&self) -> Cow<'static, str>;
+}
+
+#[cfg(feature = "stateful")]
+impl ErrorCode for NimbusError {
+    fn error_code(&self) -> Cow<'static, str> {
+        match self {
+            Self::BehaviorError(e) => format!("BehaviorError({})", e.error_code()).into(),
+            Self::ClientError(..) => "ClientError".into(),
+            Self::DatabaseNotReady => "DatabaseNotReady".into(),
+            Self::EmptyRatiosError => "EmptyRatiosError".into(),
+            Self::EvaluationError(..) => "EvaluationError".into(),
+            Self::IOError(e) => format!("IOError({:?})", e.kind()).into(),
+            Self::InternalError(..) => "InternalError".into(),
+            Self::InvalidExperimentFormat => "InvalidExperimentFormat".into(),
+            Self::InvalidExpression => "InvalidExpression".into(),
+            Self::InvalidFraction => "InvalidFraction".into(),
+            Self::InvalidPath(..) => "InvalidPath".into(),
+            Self::InvalidPersistedData => "InvalidPersistedData".into(),
+            Self::JSONError(..) => "JSONError".into(),
+            Self::NoSuchBranch(..) => "NoSuchBranch".into(),
+            Self::NoSuchExperiment(..) => "NoSuchExperiment".into(),
+            Self::OutOfBoundsError => "OutOfBoundsError".into(),
+            Self::ParseIntError(..) => "ParseIntError".into(),
+            Self::RegexError(..) => "RegexError".into(),
+            Self::RkvError(e) => format!("RkvError({})", e.error_code()).into(),
+            Self::TransformParameterError(..) => "TransformParameterError".into(),
+            Self::TryFromIntError(..) => "TryFromIntError".into(),
+            Self::TryFromSliceError(..) => "TryFromSliceError".into(),
+            Self::UniFFICallbackError(..) => "UniFFICallbackError".into(),
+            Self::UrlParsingError(..) => "UrlParsingError".into(),
+            Self::UuidError(..) => "UuidError".into(),
+            Self::VersionParsingError(..) => "VersionParsingError".into(),
+        }
+    }
+}
+
+#[cfg(feature = "stateful")]
+impl ErrorCode for rkv::StoreError {
+    fn error_code(&self) -> Cow<'static, str> {
+        match self {
+            Self::ManagerPoisonError => "ManagerPoisonError".into(),
+            Self::DatabaseCorrupted => "DatabaseCorrupted".into(),
+            Self::KeyValuePairNotFound => "KeyValuePairNotFound".into(),
+            Self::KeyValuePairBadSize => "KeyValuePairBadSize".into(),
+            Self::FileInvalid => "FileInvalid".into(),
+            Self::MapFull => "MapFull".into(),
+            Self::DbsFull => "DbsFull".into(),
+            Self::ReadersFull => "ReadersFull".into(),
+            Self::IoError(e) => format!("IoError({:?})", e.kind()).into(),
+            Self::UnsuitableEnvironmentPath(..) => "UnsuitableEnvironmentPath".into(),
+            Self::DataError(..) => "DataError".into(),
+            Self::SafeModeError(..) => "SafeModeError".into(),
+            Self::ReadTransactionAlreadyExists(..) => "ReadTransactionAlreadyExists".into(),
+            Self::OpenAttemptedDuringTransaction(..) => "OpenAttemptedDuringTransaction".into(),
+        }
+    }
+}
+
+#[cfg(feature = "stateful")]
+impl ErrorCode for BehaviorError {
+    fn error_code(&self) -> Cow<'static, str> {
+        match self {
+            Self::EventQueryParseError(..) => "EventQueryParseError",
+            Self::EventQueryTypeParseError(..) => "EventQueryTypeParseError",
+            Self::IntervalParseError(..) => "IntervalParseError",
+            Self::InvalidDuration(..) => "InvalidDuration",
+            Self::InvalidState(..) => "InvalidState",
+            Self::MissingEventStore => "MissingEventStore",
+            Self::MissingRecordedContext => "MissingRecordedContext",
+            Self::TypeError(..) => "TypeError",
+        }
+        .into()
+    }
+}

components/nimbus/src/metrics.rs

@@ -70,6 +70,10 @@ mod detail {
 
     #[uniffi::trait_interface]
     pub trait MetricsHandler: Send + Sync {
+        fn record_database_load(&self, event: DatabaseLoadExtraDef);
+
+        fn record_database_migration(&self, event: DatabaseMigrationExtraDef);
+
         fn record_enrollment_statuses(
             &self,
             enrollment_status_extras: Vec<EnrollmentStatusExtraDef>,
@@ -101,8 +105,8 @@ mod detail {
         }
     }
 
-    #[derive(Clone, Default)]
-    #[cfg_attr(test, derive(Debug, Eq, PartialEq))]
+    #[derive(Default)]
+    #[cfg_attr(test, derive(Clone, Debug, Eq, PartialEq))]
     pub struct MalformedFeatureConfigExtraDef {
         pub slug: Option<String>,
         pub branch: Option<String>,
@@ -128,6 +132,24 @@ mod detail {
             }
         }
     }
+
+    #[derive(Default)]
+    #[cfg_attr(test, derive(Clone, Debug, Eq, PartialEq))]
+    pub struct DatabaseLoadExtraDef {
+        pub corrupt: Option<bool>,
+        pub error: Option<String>,
+        pub initial_version: Option<u16>,
+        pub migrated_version: Option<u16>,
+        pub migration_error: Option<String>,
+    }
+
+    #[cfg_attr(test, derive(Clone, Debug, Eq, PartialEq))]
+    pub struct DatabaseMigrationExtraDef {
+        pub reason: String,
+        pub from_version: u16,
+        pub to_version: u16,
+        pub error: Option<String>,
+    }
 }
 
 #[cfg(not(feature = "stateful"))]

components/nimbus/src/nimbus.udl

@@ -85,6 +85,10 @@ enum EnrollmentChangeEventType {
 
 [Trait, WithForeign]
 interface MetricsHandler {
+    void record_database_load(DatabaseLoadExtraDef event);
+
+    void record_database_migration(DatabaseMigrationExtraDef event);
+
     void record_enrollment_statuses(sequence<EnrollmentStatusExtraDef> enrollment_status_extras);
     /// Feature activation is the pre-cursor to feature exposure: it is defined as the first time
     /// the feature configuration is asked for.
@@ -97,6 +101,21 @@ interface MetricsHandler {
     void submit_targeting_context();
 };
 
+dictionary DatabaseLoadExtraDef {
+    boolean? corrupt;
+    string? error;
+    u16? initial_version;
+    u16? migrated_version;
+    string? migration_error;
+};
+
+dictionary DatabaseMigrationExtraDef {
+    string reason;
+    u16 from_version;
+    u16 to_version;
+    string? error;
+};
+
 dictionary EnrollmentStatusExtraDef {
     string? branch;
     string? conflict_slug;
@@ -107,7 +126,6 @@ dictionary EnrollmentStatusExtraDef {
     sequence<PreviousGeckoPrefState>? prev_gecko_pref_states;
 };
 
-
 dictionary PreviousGeckoPrefState {
     OriginalGeckoPref original_value;
     string feature_id;

components/nimbus/src/stateful/nimbus_client.rs

@@ -25,8 +25,8 @@ use crate::evaluator::{
 };
 use crate::json::{JsonObject, PrefValue};
 use crate::metrics::{
-    EnrollmentStatusExtraDef, FeatureExposureExtraDef, MalformedFeatureConfigExtraDef,
-    MetricsHandler,
+    DatabaseLoadExtraDef, DatabaseMigrationExtraDef, EnrollmentStatusExtraDef,
+    FeatureExposureExtraDef, MalformedFeatureConfigExtraDef, MetricsHandler,
 };
 use crate::schema::parse_experiments;
 use crate::stateful::behavior::EventStore;
@@ -663,7 +663,8 @@ impl NimbusClient {
     }
 
     pub(crate) fn db(&self) -> Result<&Database> {
-        self.db.get_or_try_init(|| Database::new(&self.db_path))
+        self.db
+            .get_or_try_init(|| Database::new(&self.db_path, self.metrics_handler.clone()))
     }
 
     fn merge_additional_context(&self, context: Option<JsonObject>) -> Result<Value> {