feat: split EvaluationTrace into reason + error_code

confidence-proto/src/main/proto/confidence/telemetry.proto

@@ -5,6 +5,17 @@ option java_package = "com.spotify.telemetry.v1";
 option java_multiple_files = true;
 option java_outer_classname = "TelemetryProto";
 
+import "google/protobuf/descriptor.proto";
+
+message MetricAnnotation {
+  string name = 1;
+  string unit = 2;
+}
+
+extend google.protobuf.EnumValueOptions {
+  MetricAnnotation metric = 50641;
+}
+
 enum Platform {
   PLATFORM_UNSPECIFIED = 0;
   PLATFORM_JAVA = 1;
@@ -32,22 +43,70 @@ message LibraryTraces {
 
   message Trace {
     TraceId id = 1;
-    // only used for timed events.
-    optional uint64 millisecond_duration = 2;
+
+    oneof traceData {
+      uint64 millisecond_duration = 2 [deprecated = true];
+      RequestTrace request_trace = 3;
+      CountTrace count_trace = 4;
+      EvaluationTrace evaluation_trace = 5;
+    }
+
+    message CountTrace {}
+
+    message RequestTrace {
+      uint64 millisecond_duration = 1;
+      Status status = 2;
+
+      enum Status {
+        STATUS_UNSPECIFIED = 0;
+        STATUS_SUCCESS = 1;
+        STATUS_ERROR = 2;
+        STATUS_TIMEOUT = 3;
+        STATUS_CACHED = 4;
+      }
+    }
+
+    message EvaluationTrace {
+      EvaluationReason reason = 1;
+      EvaluationErrorCode error_code = 2;
+
+      enum EvaluationReason {
+        EVALUATION_REASON_UNSPECIFIED = 0;
+        EVALUATION_REASON_TARGETING_MATCH = 1;
+        EVALUATION_REASON_DEFAULT = 2;
+        EVALUATION_REASON_STALE = 3;
+        EVALUATION_REASON_DISABLED = 4;
+        EVALUATION_REASON_CACHED = 5;
+        EVALUATION_REASON_STATIC = 6;
+        EVALUATION_REASON_SPLIT = 7;
+        EVALUATION_REASON_ERROR = 8;
+      }
+
+      enum EvaluationErrorCode {
+        EVALUATION_ERROR_CODE_UNSPECIFIED = 0;
+        EVALUATION_ERROR_CODE_PROVIDER_NOT_READY = 1;
+        EVALUATION_ERROR_CODE_FLAG_NOT_FOUND = 2;
+        EVALUATION_ERROR_CODE_PARSE_ERROR = 3;
+        EVALUATION_ERROR_CODE_TYPE_MISMATCH = 4;
+        EVALUATION_ERROR_CODE_TARGETING_KEY_MISSING = 5;
+        EVALUATION_ERROR_CODE_INVALID_CONTEXT = 6;
+        EVALUATION_ERROR_CODE_PROVIDER_FATAL = 7;
+        EVALUATION_ERROR_CODE_GENERAL = 8;
+      }
+    }
   }
 
   enum Library {
-    LIBRARY_UNSPECIFIED = 0;
+    LIBRARY_UNKNOWN = 0;
     LIBRARY_CONFIDENCE = 1;
     LIBRARY_OPEN_FEATURE = 2;
     LIBRARY_REACT = 3;
   }
 
   enum TraceId {
     TRACE_ID_UNSPECIFIED = 0;
-    TRACE_ID_RESOLVE_LATENCY = 1;
-    TRACE_ID_STALE_FLAG = 2;
-    TRACE_ID_FLAG_TYPE_MISMATCH = 3;
-    TRACE_ID_WITH_CONTEXT = 4;
+    TRACE_ID_RESOLVE_LATENCY = 1 [(metric) = {name: "resolve_latency", unit: "ms"}];
+    TRACE_ID_STALE_FLAG = 2 [deprecated = true, (metric) = {name: "stale_flag"}];
+    TRACE_ID_FLAG_EVALUATION = 3 [(metric) = {name: "flag_evaluation"}];
   }
 }

openfeature-provider/src/main/java/com/spotify/confidence/ConfidenceFeatureProvider.java

@@ -192,13 +192,15 @@ public ProviderEvaluation<Value> getObjectEvaluation(
       }
 
       final ResolvedFlag resolvedFlag = resolveFlagResponse.getResolvedFlags(0);
+      final String reason = resolvedFlag.getReason().toString();
 
       if (resolvedFlag.getVariant().isEmpty()) {
         log.debug(
             String.format(
                 "The server returned no assignment for the flag '%s'. Typically, this happens "
                     + "if no configured rules matches the given evaluation context.",
                 flagPath.getFlag()));
+        confidence.client().trackEvaluation(reason, null);
         return ProviderEvaluation.<Value>builder()
             .value(defaultValue)
             .reason(
@@ -216,10 +218,10 @@ public ProviderEvaluation<Value> getObjectEvaluation(
           value = defaultValue;
         }
 
-        // regular resolve was successful
+        confidence.client().trackEvaluation(reason, null);
         return ProviderEvaluation.<Value>builder()
             .value(value)
-            .reason(resolvedFlag.getReason().toString())
+            .reason(reason)
             .variant(resolvedFlag.getVariant())
             .build();
       }

openfeature-provider/src/test/java/com/spotify/confidence/FeatureProviderTest.java

@@ -77,7 +77,8 @@ void beforeEach() {
     final FlagResolverClientImpl flagResolver =
         new FlagResolverClientImpl(
             new GrpcFlagResolver("fake-secret", channel, telemetryInterceptor, 1_000), telemetry);
-    final Confidence confidence = Confidence.create(fakeEventSender, flagResolver, "clientKey");
+    final Confidence confidence =
+        Confidence.create(fakeEventSender, flagResolver, "clientKey", telemetry);
     final FeatureProvider featureProvider = new ConfidenceFeatureProvider(confidence);
 
     openFeatureAPI = OpenFeatureAPI.getInstance();
@@ -525,10 +526,21 @@ public void resolvesContainHeaderWithTelemetryData() {
     assertThat(libraryTracesList).hasSize(1);
     final LibraryTraces traces = libraryTracesList.get(0);
     assertThat(traces.getLibrary()).isEqualTo(LibraryTraces.Library.LIBRARY_OPEN_FEATURE);
-    assertThat(traces.getTracesList()).hasSize(1);
-    final LibraryTraces.Trace trace = traces.getTraces(0);
-    assertThat(trace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY);
-    assertThat(trace.getMillisecondDuration()).isNotNegative();
+    assertThat(traces.getTracesList()).hasSize(2);
+    final LibraryTraces.Trace latencyTrace = traces.getTraces(0);
+    assertThat(latencyTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY);
+    assertThat(latencyTrace.getRequestTrace().getMillisecondDuration()).isNotNegative();
+    assertThat(latencyTrace.getRequestTrace().getStatus())
+        .isEqualTo(LibraryTraces.Trace.RequestTrace.Status.STATUS_SUCCESS);
+    final LibraryTraces.Trace evaluationTrace = traces.getTraces(1);
+    assertThat(evaluationTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_FLAG_EVALUATION);
+    assertThat(evaluationTrace.getEvaluationTrace().getReason())
+        .isEqualTo(
+            LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_TARGETING_MATCH);
+    assertThat(evaluationTrace.getEvaluationTrace().getErrorCode())
+        .isEqualTo(
+            LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode
+                .EVALUATION_ERROR_CODE_UNSPECIFIED);
 
     client.getIntegerDetails("flag.prop-Y", 1000, SAMPLE_CONTEXT);
   }

sdk-java/src/main/java/com/spotify/confidence/Confidence.java

@@ -128,8 +128,10 @@ public <T> FlagEvaluation<T> getEvaluation(String key, T defaultValue) {
     try {
       return getEvaluationFuture(key, defaultValue).get();
     } catch (Exception e) {
-      return new FlagEvaluation<>(
-          defaultValue, "", "ERROR", ErrorType.INTERNAL_ERROR, e.getMessage());
+      final FlagEvaluation<T> evaluation =
+          new FlagEvaluation<>(defaultValue, "", "ERROR", ErrorType.INTERNAL_ERROR, e.getMessage());
+      client().trackEvaluation(evaluation.getReason(), evaluation.getErrorType().orElse(null));
+      return evaluation;
     }
   }
 
@@ -166,8 +168,9 @@ public <T> CompletableFuture<FlagEvaluation<T>> getEvaluationFuture(String key,
                 if (resolvedFlag.getVariant().isEmpty()) {
                   final String errorMessage =
                       String.format(
-                          "The server returned no assignment for the flag '%s'. Typically, this happens "
-                              + "if no configured rules matches the given evaluation context.",
+                          "The server returned no assignment for the flag '%s'. Typically, this"
+                              + " happens if no configured rules matches the given evaluation"
+                              + " context.",
                           flagPath.getFlag());
                   log.debug(errorMessage);
                   return new FlagEvaluation<>(
@@ -196,10 +199,27 @@ public <T> CompletableFuture<FlagEvaluation<T>> getEvaluationFuture(String key,
                   }
                 }
               })
-          .exceptionally(handleFlagEvaluationError(defaultValue));
+          .thenApply(
+              evaluation -> {
+                client()
+                    .trackEvaluation(
+                        evaluation.getReason(), evaluation.getErrorType().orElse(null));
+                return evaluation;
+              })
+          .exceptionally(
+              e -> {
+                final FlagEvaluation<T> evaluation =
+                    handleFlagEvaluationError(defaultValue).apply(e);
+                client()
+                    .trackEvaluation(
+                        evaluation.getReason(), evaluation.getErrorType().orElse(null));
+                return evaluation;
+              });
 
     } catch (Exception e) {
-      return CompletableFuture.completedFuture(handleFlagEvaluationError(defaultValue).apply(e));
+      final FlagEvaluation<T> evaluation = handleFlagEvaluationError(defaultValue).apply(e);
+      client().trackEvaluation(evaluation.getReason(), evaluation.getErrorType().orElse(null));
+      return CompletableFuture.completedFuture(evaluation);
     }
   }
 
@@ -218,7 +238,8 @@ public void logResolveTesterHint(ResolvedFlag resolvedFlag) {
           Base64.getEncoder().encodeToString(jsonPrinter.print(resolveTesterLogging).getBytes());
       final String logMessage =
           String.format(
-              "Check your flag evaluation for '%s' by copy pasting the payload to the Resolve tester '%s'",
+              "Check your flag evaluation for '%s' by copy pasting the payload to the Resolve"
+                  + " tester '%s'",
               flag, base64);
       log.debug(logMessage);
     } catch (InvalidProtocolBufferException e) {
@@ -236,11 +257,20 @@ static Confidence create(
       EventSenderEngine eventSenderEngine,
       FlagResolverClient flagResolverClient,
       String clientSecret) {
+    return create(eventSenderEngine, flagResolverClient, clientSecret, null);
+  }
+
+  @VisibleForTesting
+  static Confidence create(
+      EventSenderEngine eventSenderEngine,
+      FlagResolverClient flagResolverClient,
+      String clientSecret,
+      @Nullable Telemetry telemetry) {
     final Closer closer = Closer.create();
     closer.register(eventSenderEngine);
     closer.register(flagResolverClient);
     return new RootInstance(
-        new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret));
+        new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret, telemetry));
   }
 
   public static Confidence.Builder builder(String clientSecret) {
@@ -252,16 +282,19 @@ static class ClientDelegate implements FlagResolverClient, EventSenderEngine {
     private final FlagResolverClient flagResolverClient;
     private final EventSenderEngine eventSenderEngine;
     private String clientSecret;
+    @Nullable private final Telemetry telemetry;
 
     ClientDelegate(
         Closeable closeable,
         FlagResolverClient flagResolverClient,
         EventSenderEngine eventSenderEngine,
-        String clientSecret) {
+        String clientSecret,
+        @Nullable Telemetry telemetry) {
       this.closeable = closeable;
       this.flagResolverClient = flagResolverClient;
       this.eventSenderEngine = eventSenderEngine;
       this.clientSecret = clientSecret;
+      this.telemetry = telemetry;
     }
 
     @Override
@@ -281,6 +314,14 @@ public CompletableFuture<ResolveFlagsResponse> resolveFlags(
       return flagResolverClient.resolveFlags(flag, context);
     }
 
+    void trackEvaluation(String resolveReason, @Nullable ErrorType errorType) {
+      if (telemetry != null) {
+        telemetry.appendEvaluation(
+            Telemetry.mapReason(resolveReason, errorType),
+            Telemetry.mapErrorCode(resolveReason, errorType));
+      }
+    }
+
     @Override
     public void close() throws IOException {
       closeable.close();
@@ -417,7 +458,8 @@ public Confidence build() {
       closer.register(flagResolverClient);
       closer.register(eventSenderEngine);
       return new RootInstance(
-          new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret));
+          new ClientDelegate(
+              closer, flagResolverClient, eventSenderEngine, clientSecret, telemetry));
     }
 
     private void registerChannelForShutdown(ManagedChannel channel) {

sdk-java/src/main/java/com/spotify/confidence/ConfidenceStub.java

@@ -161,7 +161,7 @@ public List<String> getCallHistory() {
   // Mock implementation of ClientDelegate
   private static class MockClientDelegate extends ClientDelegate {
     private MockClientDelegate() {
-      super(null, null, null, "");
+      super(null, null, null, "", null);
     }
 
     @Override