PYCBC-1718: Update metrics, orphan reporting and tracing config to align with C++ updates

Changes
========
* Update C++ core to pull in changes to metrics, orphan reporting and
  tracing
* Add `ClusterMetricsOptions` and `ClusterOrphanReportingOptions` options
  blocks
* Add deprecated notice to `ClusterTracingOptions`
  `tracing_orphaned_queue_size` and
  `tracing_orphaned_queue_flush_interval` options as those are now in
  `ClusterOrphanReportingOptions`
* Add deprecated notice to `ClusterOptions` `logging_meter_emit_interval`
  as that option is now in `ClusterMetricsOptions`
* Update cluster option processing logic to handle updated options
  blocks (and option values being able to come from multiple blocks)
* Update connection tests to confirm new functionality
* Update tracer tests to handle changes to C++ core

Results
=======
All tests pass.

Change-Id: I65e65a3357498f8d0d0746599fcab3e8caf5d3cd
Reviewed-on: https://review.couchbase.org/c/couchbase-python-client/+/235475
Reviewed-by: Dimitris Christodoulou <dimitris.christodoulou@couchbase.com>
Tested-by: Build Bot <build@couchbase.com>

Author: thejcfactor

couchbase/logic/cluster.py

@@ -30,7 +30,9 @@
 from couchbase.auth import CertificateAuthenticator, PasswordAuthenticator
 from couchbase.diagnostics import ServiceType
 from couchbase.exceptions import InvalidArgumentException
-from couchbase.options import (ClusterOptions,
+from couchbase.options import (ClusterMetricsOptions,
+                               ClusterOptions,
+                               ClusterOrphanReportingOptions,
                                ClusterTimeoutOptions,
                                ClusterTracingOptions,
                                TLSVerifyMode,
@@ -144,9 +146,48 @@ def __init__(self,  # noqa: C901
         for key in ClusterTracingOptions.get_allowed_option_keys(use_transform_keys=True):
             if key in cluster_opts:
                 tracing_opts[key] = cluster_opts.pop(key)
+
+        orphan_opts = {}
+        for key in ClusterOrphanReportingOptions.get_allowed_option_keys(use_transform_keys=True):
+            if key in cluster_opts:
+                orphan_opts[key] = cluster_opts.pop(key)
+
+        # PYCBC-XXXX: C++ core split out orphan reporting from tracing, so we have separate options blocks now.
+        #           However, we have to check for the old options in the tracing block for backwards compatibility.
+        if tracing_opts:
+            sample_size = tracing_opts.pop('orphan_sample_size', None)
+            if sample_size and 'sample_size' not in orphan_opts:
+                orphan_opts['orphan_sample_size'] = sample_size
+            emit_interval = tracing_opts.pop('orphan_emit_interval', None)
+            if emit_interval and 'emit_interval' not in orphan_opts:
+                orphan_opts['orphan_emit_interval'] = emit_interval
+
+        cluster_enable_tracing = cluster_opts.pop('cluster_enable_tracing', None)
+        if cluster_enable_tracing is not None and 'enable_tracing' not in tracing_opts:
+            tracing_opts['enable_tracing'] = cluster_enable_tracing
         if tracing_opts:
             cluster_opts['tracing_options'] = tracing_opts
 
+        cluster_enable_orphan_reporting = cluster_opts.pop('cluster_enable_orphan_reporting', None)
+        if cluster_enable_orphan_reporting is not None and 'enable_orphan_reporting' not in orphan_opts:
+            orphan_opts['enable_orphan_reporting'] = cluster_enable_orphan_reporting
+        if orphan_opts:
+            cluster_opts['orphan_reporting_options'] = orphan_opts
+
+        metrics_opts = {}
+        for key in ClusterMetricsOptions.get_allowed_option_keys(use_transform_keys=True):
+            if key in cluster_opts:
+                metrics_opts[key] = cluster_opts.pop(key)
+
+        cluster_metrics_emit_interval = cluster_opts.pop('cluster_metrics_emit_interval', None)
+        if cluster_metrics_emit_interval and 'metrics_emit_interval' not in metrics_opts:
+            metrics_opts['metrics_emit_interval'] = cluster_metrics_emit_interval
+        cluster_enable_metrics = cluster_opts.pop('cluster_enable_metrics', None)
+        if cluster_enable_metrics is not None and 'enable_metrics' not in metrics_opts:
+            metrics_opts['enable_metrics'] = cluster_enable_metrics
+        if metrics_opts:
+            cluster_opts['metrics_options'] = metrics_opts
+
         self._default_serializer = cluster_opts.pop("serializer", None)
         if not self._default_serializer:
             self._default_serializer = DefaultJsonSerializer()

couchbase/logic/options.py

@@ -454,9 +454,102 @@ def get_allowed_option_keys(use_transform_keys=False  # type: Optional[bool]
         return list(ClusterTimeoutOptionsBase._VALID_OPTS.keys())
 
 
+class ClusterMetricsOptionsBase(dict):
+
+    _VALID_OPTS = {
+        "metrics_enable_metrics": {"enable_metrics": validate_bool},
+        "metrics_emit_interval": {"metrics_emit_interval": timedelta_as_microseconds}
+    }
+
+    @overload
+    def __init__(
+        self,
+        enable_metrics=None,  # type: Optional[bool]
+        emit_interval=None,  # type: Optional[timedelta]
+    ):
+        """ClusterMetricsOptions instance."""
+
+    def __init__(self, **kwargs):
+        kwargs = {f'metrics_{k}': v for k, v in kwargs.items() if v is not None}
+        super().__init__(**kwargs)
+
+    def as_dict(self):
+        opts = {}
+        allowed_opts = ClusterMetricsOptionsBase.get_allowed_option_keys()
+        for k, v in self.items():
+            if k not in allowed_opts:
+                continue
+            if v is None:
+                continue
+            if isinstance(v, timedelta):
+                opts[k] = v.total_seconds()
+            elif isinstance(v, (int, float)):
+                opts[k] = v
+        return opts
+
+    @staticmethod
+    def get_allowed_option_keys(use_transform_keys=False  # type: Optional[bool]
+                                ) -> List[str]:
+        if use_transform_keys is True:
+            keys = []
+            for val in ClusterMetricsOptionsBase._VALID_OPTS.values():
+                keys.append(list(val.keys())[0])
+            return keys
+
+        return list(ClusterMetricsOptionsBase._VALID_OPTS.keys())
+
+
+class ClusterOrphanReportingOptionsBase(dict):
+
+    _VALID_OPTS = {
+        "orphan_enable_orphan_reporting": {"enable_orphan_reporting": validate_bool},
+        "orphan_sample_size": {"orphan_sample_size": validate_int},
+        "orphan_emit_interval": {"orphan_emit_interval": timedelta_as_microseconds}
+    }
+
+    @overload
+    def __init__(
+        self,
+        enable_orphan_reporting=None,  # type: Optional[bool]
+        sample_size=None,  # type: Optional[int]
+        emit_interval=None,  # type: Optional[timedelta]
+    ):
+        """ClusterOrphanReportingOptions instance."""
+
+    def __init__(self, **kwargs):
+        kwargs = {f'orphan_{k}': v for k, v in kwargs.items() if v is not None}
+        super().__init__(**kwargs)
+
+    def as_dict(self):
+        opts = {}
+        allowed_opts = ClusterOrphanReportingOptionsBase.get_allowed_option_keys()
+        for k, v in self.items():
+            if k not in allowed_opts:
+                continue
+            if v is None:
+                continue
+            if isinstance(v, timedelta):
+                opts[k] = v.total_seconds()
+            elif isinstance(v, (int, float)):
+                opts[k] = v
+        return opts
+
+    @staticmethod
+    def get_allowed_option_keys(use_transform_keys=False  # type: Optional[bool]
+                                ) -> List[str]:
+        if use_transform_keys is True:
+            keys = []
+            for val in ClusterOrphanReportingOptionsBase._VALID_OPTS.values():
+                keys.append(list(val.keys())[0])
+            return keys
+
+        return list(ClusterOrphanReportingOptionsBase._VALID_OPTS.keys())
+
+
 class ClusterTracingOptionsBase(dict):
 
     _VALID_OPTS = {
+        "tracing_enable_tracing": {"enable_tracing": validate_bool},
         "tracing_threshold_kv": {"key_value_threshold": timedelta_as_microseconds},
         "tracing_threshold_view": {"view_threshold": timedelta_as_microseconds},
         "tracing_threshold_query": {"query_threshold": timedelta_as_microseconds},
@@ -466,13 +559,14 @@ class ClusterTracingOptionsBase(dict):
         "tracing_threshold_management": {"management_threshold": timedelta_as_microseconds},
         "tracing_threshold_queue_size": {"threshold_sample_size": validate_int},
         "tracing_threshold_queue_flush_interval": {"threshold_emit_interval": timedelta_as_microseconds},
-        "tracing_orphaned_queue_size": {"orphaned_sample_size": validate_int},
-        "tracing_orphaned_queue_flush_interval": {"orphaned_emit_interval": timedelta_as_microseconds}
+        "tracing_orphaned_queue_size": {"orphan_sample_size": validate_int},
+        "tracing_orphaned_queue_flush_interval": {"orphan_emit_interval": timedelta_as_microseconds}
     }
 
     @overload
     def __init__(
         self,
+        enable_tracing=None,  # type: Optional[bool]
         tracing_threshold_kv=None,  # type: Optional[timedelta]
         tracing_threshold_view=None,  # type: Optional[timedelta]
         tracing_threshold_query=None,  # type: Optional[timedelta]
@@ -489,6 +583,9 @@ def __init__(
 
     def __init__(self, **kwargs):
         kwargs = {k: v for k, v in kwargs.items() if v is not None}
+        enable_tracing = kwargs.pop('enable_tracing', None)
+        if enable_tracing is not None:
+            kwargs['tracing_enable_tracing'] = enable_tracing
         super().__init__(**kwargs)
 
     def as_dict(self):
@@ -535,8 +632,9 @@ class ClusterOptionsBase(dict):
         "enable_unordered_execution": {"enable_unordered_execution": validate_bool},
         "enable_clustermap_notification": {"enable_clustermap_notification": validate_bool},
         "enable_compression": {"enable_compression": validate_bool},
-        "enable_tracing": {"enable_tracing": validate_bool},
-        "enable_metrics": {"enable_metrics": validate_bool},
+        "enable_tracing": {"cluster_enable_tracing": validate_bool},
+        "enable_metrics": {"cluster_enable_metrics": validate_bool},
+        "enable_orphan_reporting": {"cluster_enable_orphan_reporting": validate_bool},
         "network": {"network": validate_str},
         "tls_verify": {"tls_verify": TLSVerifyMode.to_str},
         "serializer": {"serializer": lambda x: x},
@@ -550,7 +648,7 @@ class ClusterOptionsBase(dict):
         "trust_store_path": {"trust_store_path": validate_str},
         "cert_path": {"cert_path": validate_str},
         "disable_mozilla_ca_certificates": {"disable_mozilla_ca_certificates": validate_bool},
-        "logging_meter_emit_interval": {"emit_interval": timedelta_as_microseconds},
+        "logging_meter_emit_interval": {"cluster_metrics_emit_interval": timedelta_as_microseconds},
         "num_io_threads": {"num_io_threads": validate_int},
         "transaction_config": {"transaction_config": lambda x: x},
         "tracer": {"tracer": lambda x: x},
@@ -572,6 +670,8 @@ def __init__(
         self,
         authenticator,  # type: Authenticator
         timeout_options=None,  # type: Optional[ClusterTimeoutOptionsBase]
+        orphan_reporting_options=None,  # type: Optional[ClusterOrphanReportingOptionsBase]
+        metrics_options=None,  # type: Optional[ClusterMetricsOptionsBase]
         tracing_options=None,  # type: Optional[ClusterTracingOptionsBase]
         enable_tls=None,    # type: Optional[bool]
         enable_mutation_tokens=None,    # type: Optional[bool]
@@ -584,6 +684,7 @@ def __init__(
         enable_compression=None,    # type: Optional[bool]
         enable_tracing=None,    # type: Optional[bool]
         enable_metrics=None,    # type: Optional[bool]
+        enable_orphan_reporting=None,    # type: Optional[bool]
         network=None,    # type: Optional[str]
         tls_verify=None,    # type: Optional[Union[TLSVerifyMode, str]]
         serializer=None,  # type: Optional[Serializer]
@@ -616,7 +717,7 @@ def __init__(
     ):
         """ClusterOptions instance."""
 
-    def __init__(self,
+    def __init__(self,  # noqa: C901
                  authenticator,  # type: Authenticator
                  **kwargs
                  ):
@@ -631,6 +732,18 @@ def __init__(self,
                 if k not in kwargs:
                     kwargs[k] = v
 
+        orphan_opts = kwargs.pop('orphan_reporting_options', {})
+        if orphan_opts:
+            for k, v in orphan_opts.items():
+                if k not in kwargs:
+                    kwargs[k] = v
+
+        metrics_opts = kwargs.pop('metrics_options', {})
+        if metrics_opts:
+            for k, v in metrics_opts.items():
+                if k not in kwargs:
+                    kwargs[k] = v
+
         timeout_opts = kwargs.pop('timeout_options', {})
         if timeout_opts:
             for k, v in timeout_opts.items():
@@ -654,13 +767,17 @@ def get_allowed_option_keys(cluster_opts_only=False,  # type: Optional[bool]
 
             keys.extend(ClusterTimeoutOptionsBase.get_allowed_option_keys(use_transform_keys=True))
             keys.extend(ClusterTracingOptionsBase.get_allowed_option_keys(use_transform_keys=True))
+            keys.extend(ClusterMetricsOptionsBase.get_allowed_option_keys(use_transform_keys=True))
+            keys.extend(ClusterOrphanReportingOptionsBase.get_allowed_option_keys(use_transform_keys=True))
 
             return keys
 
         if cluster_opts_only is True:
             return list(ClusterOptionsBase._VALID_OPTS.keys())
 
         valid_keys = ClusterTimeoutOptionsBase.get_allowed_option_keys()
+        valid_keys.extend(ClusterOrphanReportingOptionsBase.get_allowed_option_keys())
+        valid_keys.extend(ClusterMetricsOptionsBase.get_allowed_option_keys())
         valid_keys.extend(ClusterTracingOptionsBase.get_allowed_option_keys())
         valid_keys.extend(list(ClusterOptionsBase._VALID_OPTS.keys()))
 
@@ -669,6 +786,8 @@ def get_allowed_option_keys(cluster_opts_only=False,  # type: Optional[bool]
     @staticmethod
     def get_valid_options() -> Dict[str, Any]:
         valid_opts = copy.copy(ClusterTimeoutOptionsBase._VALID_OPTS)
+        valid_opts.update(copy.copy(ClusterMetricsOptionsBase._VALID_OPTS))
+        valid_opts.update(copy.copy(ClusterOrphanReportingOptionsBase._VALID_OPTS))
         valid_opts.update(copy.copy(ClusterTracingOptionsBase._VALID_OPTS))
         valid_opts.update(copy.copy(ClusterOptionsBase._VALID_OPTS))
         return valid_opts