chore: refactor otel integration (#2794)

* chore: refactor otel exporter

Previously bigtable client would construct 2 otel instances:
- one for internal metrics not relevant for customers
- another for user visible metrics

The 2nd otel instance would be used for both exporting to cloud monitoring and to end user provided exporters.
This was done by having the enduser provide a SdkMeterProviderBuilder and use a utility to inject the bigtable builtin exporter.
This created a number of problems and overheads.

This PR changes the split, now there is a builtin otel instance with a bigtable exporter preconfigured. This receives both internal and user visible cloud monitoring metrics.
This exporter can be disabled via EnhancedStubSettings#disableInternalMetrics()

In addition, if an enduser provides their own otel instance via CustomOpenTelemetryMetricsProvider, this will be dual written in parallel to the builtin one.
Now the default case is that we have 1 otel instance and we dont have to inject our internal exporter into the use otel instance.

Change-Id: Ic0b8efdcdf342aef8f589c2c9e05b7672a0e5d08

* combine the exporters

Change-Id: Id8a2052a4d49a9623a2b63debe1f8ef10210f84c

Author: igorbernstein2

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java

@@ -111,7 +111,9 @@ public BigtableDataClient createDefault() {
           sharedClientContext.getClientContext().toBuilder()
               .setTracerFactory(
                   EnhancedBigtableStub.createBigtableTracerFactory(
-                      defaultSettings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                      defaultSettings.getStubSettings(),
+                      sharedClientContext.getBuiltinOpenTelemetry(),
+                      sharedClientContext.getUserOpenTelemetry()))
               .build();
 
       return BigtableDataClient.createWithClientContext(
@@ -140,7 +142,9 @@ public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) thro
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getBuiltinOpenTelemetry(),
+                    sharedClientContext.getUserOpenTelemetry()))
             .build();
     return BigtableDataClient.createWithClientContext(
         settings, sharedClientContext.withClientContext(clientContext));
@@ -168,7 +172,9 @@ public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getBuiltinOpenTelemetry(),
+                    sharedClientContext.getUserOpenTelemetry()))
             .build();
 
     return BigtableDataClient.createWithClientContext(
@@ -197,7 +203,9 @@ public BigtableDataClient createForInstance(
         sharedClientContext.getClientContext().toBuilder()
             .setTracerFactory(
                 EnhancedBigtableStub.createBigtableTracerFactory(
-                    settings.getStubSettings(), sharedClientContext.getOpenTelemetry()))
+                    settings.getStubSettings(),
+                    sharedClientContext.getBuiltinOpenTelemetry(),
+                    sharedClientContext.getUserOpenTelemetry()))
             .build();
     return BigtableDataClient.createWithClientContext(
         settings, sharedClientContext.withClientContext(clientContext));

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/BigtableClientContext.java

@@ -26,14 +26,12 @@
 import com.google.api.gax.rpc.ClientContext;
 import com.google.auth.Credentials;
 import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials;
-import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.bigtable.v2.InstanceName;
 import com.google.cloud.bigtable.data.v2.internal.JwtCredentialsWithAudience;
 import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants;
 import com.google.cloud.bigtable.data.v2.stub.metrics.ChannelPoolMetricsTracer;
 import com.google.cloud.bigtable.data.v2.stub.metrics.CustomOpenTelemetryMetricsProvider;
-import com.google.cloud.bigtable.data.v2.stub.metrics.DefaultMetricsProvider;
-import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsProvider;
-import com.google.cloud.bigtable.data.v2.stub.metrics.NoopMetricsProvider;
+import com.google.cloud.bigtable.data.v2.stub.metrics.Util;
 import com.google.cloud.bigtable.gaxx.grpc.BigtableTransportChannelProvider;
 import com.google.cloud.bigtable.gaxx.grpc.ChannelPrimer;
 import io.grpc.ManagedChannelBuilder;
@@ -57,9 +55,8 @@ public class BigtableClientContext {
 
   private static final Logger logger = Logger.getLogger(BigtableClientContext.class.getName());
 
-  @Nullable private final OpenTelemetry openTelemetry;
-  @Nullable private final OpenTelemetrySdk internalOpenTelemetry;
-  private final MetricsProvider metricsProvider;
+  @Nullable private final OpenTelemetrySdk builtinOpenTelemetry;
+  @Nullable private final OpenTelemetry userOpenTelemetry;
   private final ClientContext clientContext;
   // the background executor shared for OTEL instances and monitoring client and all other
   // background tasks
@@ -89,17 +86,24 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
     builder.setBackgroundExecutorProvider(executorProvider);
 
     // Set up OpenTelemetry
-    OpenTelemetry openTelemetry = null;
+    @Nullable OpenTelemetry userOtel = null;
+    if (settings.getMetricsProvider() instanceof CustomOpenTelemetryMetricsProvider) {
+      userOtel =
+          ((CustomOpenTelemetryMetricsProvider) settings.getMetricsProvider()).getOpenTelemetry();
+    }
+
+    @Nullable OpenTelemetrySdk builtinOtel = null;
     try {
-      // We don't want client side metrics to crash the client, so catch any exception when getting
-      // the OTEL instance and log the exception instead.
-      openTelemetry =
-          getOpenTelemetryFromMetricsProvider(
-              settings.getMetricsProvider(),
-              credentials,
-              settings.getMetricsEndpoint(),
-              universeDomain,
-              backgroundExecutor);
+      if (settings.areInternalMetricsEnabled()) {
+        builtinOtel =
+            Util.createBuiltinOtel(
+                InstanceName.of(settings.getProjectId(), settings.getInstanceId()),
+                settings.getAppProfileId(),
+                credentials,
+                settings.getMetricsEndpoint(),
+                universeDomain,
+                backgroundExecutor);
+      }
     } catch (Throwable t) {
       logger.log(Level.WARNING, "Failed to get OTEL, will skip exporting client side metrics", t);
     }
@@ -110,21 +114,16 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
             ? ((InstantiatingGrpcChannelProvider) builder.getTransportChannelProvider()).toBuilder()
             : null;
 
-    @Nullable OpenTelemetrySdk internalOtel = null;
     @Nullable ChannelPoolMetricsTracer channelPoolMetricsTracer = null;
     // Internal metrics are scoped to the connections, so we need a mutable transportProvider,
     // otherwise there is
     // no reason to build the internal OtelProvider
     if (transportProvider != null) {
-      internalOtel =
-          settings
-              .getInternalMetricsProvider()
-              .createOtelProvider(settings, credentials, backgroundExecutor);
-      if (internalOtel != null) {
-        channelPoolMetricsTracer = new ChannelPoolMetricsTracer(internalOtel);
+      if (builtinOtel != null) {
+        channelPoolMetricsTracer = new ChannelPoolMetricsTracer(builtinOtel);
 
         // Configure grpc metrics
-        configureGrpcOtel(transportProvider, internalOtel);
+        configureGrpcOtel(transportProvider, builtinOtel);
       }
     }
 
@@ -149,7 +148,7 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
 
       BigtableTransportChannelProvider btTransportProvider =
           BigtableTransportChannelProvider.create(
-              (InstantiatingGrpcChannelProvider) transportProvider.build(),
+              transportProvider.build(),
               channelPrimer,
               channelPoolMetricsTracer,
               backgroundExecutor);
@@ -162,12 +161,7 @@ public static BigtableClientContext create(EnhancedBigtableStubSettings settings
       channelPoolMetricsTracer.start(clientContext.getExecutor());
     }
 
-    return new BigtableClientContext(
-        clientContext,
-        openTelemetry,
-        internalOtel,
-        settings.getMetricsProvider(),
-        executorProvider);
+    return new BigtableClientContext(clientContext, builtinOtel, userOtel, executorProvider);
   }
 
   private static void configureGrpcOtel(
@@ -199,19 +193,23 @@ private static void configureGrpcOtel(
 
   private BigtableClientContext(
       ClientContext clientContext,
-      @Nullable OpenTelemetry openTelemetry,
       @Nullable OpenTelemetrySdk internalOtel,
-      MetricsProvider metricsProvider,
+      @Nullable OpenTelemetry userOpenTelemetry,
       ExecutorProvider backgroundExecutorProvider) {
     this.clientContext = clientContext;
-    this.openTelemetry = openTelemetry;
-    this.internalOpenTelemetry = internalOtel;
-    this.metricsProvider = metricsProvider;
+    this.userOpenTelemetry = userOpenTelemetry;
+    this.builtinOpenTelemetry = internalOtel;
     this.backgroundExecutorProvider = backgroundExecutorProvider;
   }
 
-  public OpenTelemetry getOpenTelemetry() {
-    return this.openTelemetry;
+  @Nullable
+  public OpenTelemetrySdk getBuiltinOpenTelemetry() {
+    return builtinOpenTelemetry;
+  }
+
+  @Nullable
+  public OpenTelemetry getUserOpenTelemetry() {
+    return this.userOpenTelemetry;
   }
 
   public ClientContext getClientContext() {
@@ -220,53 +218,24 @@ public ClientContext getClientContext() {
 
   public BigtableClientContext withClientContext(ClientContext clientContext) {
     return new BigtableClientContext(
-        clientContext,
-        openTelemetry,
-        internalOpenTelemetry,
-        metricsProvider,
-        backgroundExecutorProvider);
+        clientContext, builtinOpenTelemetry, userOpenTelemetry, backgroundExecutorProvider);
   }
 
   public void close() throws Exception {
     for (BackgroundResource resource : clientContext.getBackgroundResources()) {
       resource.close();
     }
-    if (internalOpenTelemetry != null) {
-      internalOpenTelemetry.close();
+    if (builtinOpenTelemetry != null) {
+      builtinOpenTelemetry.close();
     }
-    if (metricsProvider instanceof DefaultMetricsProvider && openTelemetry != null) {
-      ((OpenTelemetrySdk) openTelemetry).close();
+    if (builtinOpenTelemetry != null) {
+      builtinOpenTelemetry.close();
     }
     if (backgroundExecutorProvider.shouldAutoClose()) {
       backgroundExecutorProvider.getExecutor().shutdown();
     }
   }
 
-  private static OpenTelemetry getOpenTelemetryFromMetricsProvider(
-      MetricsProvider metricsProvider,
-      @Nullable Credentials defaultCredentials,
-      @Nullable String metricsEndpoint,
-      String universeDomain,
-      ScheduledExecutorService executor)
-      throws IOException {
-    if (metricsProvider instanceof CustomOpenTelemetryMetricsProvider) {
-      CustomOpenTelemetryMetricsProvider customMetricsProvider =
-          (CustomOpenTelemetryMetricsProvider) metricsProvider;
-      return customMetricsProvider.getOpenTelemetry();
-    } else if (metricsProvider instanceof DefaultMetricsProvider) {
-      Credentials credentials =
-          BigtableDataSettings.getMetricsCredentials() != null
-              ? BigtableDataSettings.getMetricsCredentials()
-              : defaultCredentials;
-      DefaultMetricsProvider defaultMetricsProvider = (DefaultMetricsProvider) metricsProvider;
-      return defaultMetricsProvider.getOpenTelemetry(
-          metricsEndpoint, universeDomain, credentials, executor);
-    } else if (metricsProvider instanceof NoopMetricsProvider) {
-      return null;
-    }
-    throw new IOException("Invalid MetricsProvider type " + metricsProvider);
-  }
-
   private static void patchCredentials(EnhancedBigtableStubSettings.Builder settings)
       throws IOException {
     String audience = settings.getJwtAudience();

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java

@@ -200,10 +200,13 @@ public class EnhancedBigtableStub implements AutoCloseable {
   public static EnhancedBigtableStub create(EnhancedBigtableStubSettings settings)
       throws IOException {
     BigtableClientContext bigtableClientContext = createBigtableClientContext(settings);
-    OpenTelemetry openTelemetry = bigtableClientContext.getOpenTelemetry();
     ClientContext contextWithTracer =
         bigtableClientContext.getClientContext().toBuilder()
-            .setTracerFactory(createBigtableTracerFactory(settings, openTelemetry))
+            .setTracerFactory(
+                createBigtableTracerFactory(
+                    settings,
+                    bigtableClientContext.getBuiltinOpenTelemetry(),
+                    bigtableClientContext.getUserOpenTelemetry()))
             .build();
     bigtableClientContext = bigtableClientContext.withClientContext(contextWithTracer);
     return new EnhancedBigtableStub(settings, bigtableClientContext);
@@ -222,18 +225,21 @@ public static BigtableClientContext createBigtableClientContext(
   }
 
   public static ApiTracerFactory createBigtableTracerFactory(
-      EnhancedBigtableStubSettings settings, @Nullable OpenTelemetry openTelemetry)
+      EnhancedBigtableStubSettings settings,
+      @Nullable OpenTelemetry builtinOtel,
+      @Nullable OpenTelemetry userOtel)
       throws IOException {
     return createBigtableTracerFactory(
-        settings, Tags.getTagger(), Stats.getStatsRecorder(), openTelemetry);
+        settings, Tags.getTagger(), Stats.getStatsRecorder(), builtinOtel, userOtel);
   }
 
   @VisibleForTesting
   public static ApiTracerFactory createBigtableTracerFactory(
       EnhancedBigtableStubSettings settings,
       Tagger tagger,
       StatsRecorder stats,
-      @Nullable OpenTelemetry openTelemetry)
+      @Nullable OpenTelemetry builtinOtel,
+      @Nullable OpenTelemetry userOtel)
       throws IOException {
     String projectId = settings.getProjectId();
     String instanceId = settings.getInstanceId();
@@ -265,12 +271,14 @@ public static ApiTracerFactory createBigtableTracerFactory(
         .add(MetricsTracerFactory.create(tagger, stats, attributes))
         // Add user configured tracer
         .add(settings.getTracerFactory());
-    BuiltinMetricsTracerFactory builtinMetricsTracerFactory =
-        openTelemetry != null
-            ? BuiltinMetricsTracerFactory.create(openTelemetry, createBuiltinAttributes(settings))
-            : null;
-    if (builtinMetricsTracerFactory != null) {
-      tracerFactories.add(builtinMetricsTracerFactory);
+
+    if (builtinOtel != null) {
+      tracerFactories.add(
+          BuiltinMetricsTracerFactory.create(builtinOtel, createBuiltinAttributes(settings)));
+    }
+    if (userOtel != null) {
+      tracerFactories.add(
+          BuiltinMetricsTracerFactory.create(userOtel, createBuiltinAttributes(settings)));
     }
     return new CompositeTracerFactory(tracerFactories.build());
   }