Add buffer support to remaining Observability Pipelines destinations

.generator/schemas/v2/openapi.yaml

@@ -52162,6 +52162,8 @@ components:
           description: S3 bucket name.
           example: "my-bucket"
           type: string
+        buffer:
+          $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         compression:
           $ref: "#/components/schemas/ObservabilityPipelineAmazonS3GenericCompression"
         encoding:
@@ -52411,6 +52413,8 @@ components:
 
         **Supported pipeline types:** logs
       properties:
+        buffer:
+          $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         endpoint_url_key:
           description: Name of the environment variable or secret that holds the CloudPrem endpoint URL.
           example: CLOUDPREM_ENDPOINT_URL
@@ -54061,6 +54065,8 @@ components:
       properties:
         auth_strategy:
           $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationAuthStrategy"
+        buffer:
+          $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         compression:
           $ref: "#/components/schemas/ObservabilityPipelineHttpClientDestinationCompression"
         custom_key:
@@ -54345,6 +54351,8 @@ components:
           description: Name of the environment variable or secret that holds the Kafka bootstrap servers list.
           example: KAFKA_BOOTSTRAP_SERVERS
           type: string
+        buffer:
+          $ref: "#/components/schemas/ObservabilityPipelineBufferOptions"
         compression:
           $ref: "#/components/schemas/ObservabilityPipelineKafkaDestinationCompression"
         encoding:

examples/v2/observability-pipelines/ValidatePipeline_1070998129.py

@@ -0,0 +1,95 @@
+"""
+Validate an observability pipeline with cloud_prem destination buffer returns "OK" response
+"""
+
+from datadog_api_client import ApiClient, Configuration
+from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
+from datadog_api_client.v2.model.observability_pipeline_buffer_options_disk_type import (
+    ObservabilityPipelineBufferOptionsDiskType,
+)
+from datadog_api_client.v2.model.observability_pipeline_buffer_options_when_full import (
+    ObservabilityPipelineBufferOptionsWhenFull,
+)
+from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination import (
+    ObservabilityPipelineCloudPremDestination,
+)
+from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import (
+    ObservabilityPipelineCloudPremDestinationType,
+)
+from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
+from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
+    ObservabilityPipelineConfigProcessorGroup,
+)
+from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
+from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
+    ObservabilityPipelineDatadogAgentSource,
+)
+from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import (
+    ObservabilityPipelineDatadogAgentSourceType,
+)
+from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import (
+    ObservabilityPipelineDiskBufferOptions,
+)
+from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor
+from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import (
+    ObservabilityPipelineFilterProcessorType,
+)
+from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec
+from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData
+
+body = ObservabilityPipelineSpec(
+    data=ObservabilityPipelineSpecData(
+        attributes=ObservabilityPipelineDataAttributes(
+            config=ObservabilityPipelineConfig(
+                destinations=[
+                    ObservabilityPipelineCloudPremDestination(
+                        id="cloud-prem-destination",
+                        inputs=[
+                            "my-processor-group",
+                        ],
+                        type=ObservabilityPipelineCloudPremDestinationType.CLOUD_PREM,
+                        endpoint_url_key="CLOUDPREM_ENDPOINT_URL",
+                        buffer=ObservabilityPipelineDiskBufferOptions(
+                            type=ObservabilityPipelineBufferOptionsDiskType.DISK,
+                            max_size=1073741824,
+                            when_full=ObservabilityPipelineBufferOptionsWhenFull.BLOCK,
+                        ),
+                    ),
+                ],
+                processor_groups=[
+                    ObservabilityPipelineConfigProcessorGroup(
+                        enabled=True,
+                        id="my-processor-group",
+                        include="service:my-service",
+                        inputs=[
+                            "datadog-agent-source",
+                        ],
+                        processors=[
+                            ObservabilityPipelineFilterProcessor(
+                                enabled=True,
+                                id="filter-processor",
+                                include="status:error",
+                                type=ObservabilityPipelineFilterProcessorType.FILTER,
+                            ),
+                        ],
+                    ),
+                ],
+                sources=[
+                    ObservabilityPipelineDatadogAgentSource(
+                        id="datadog-agent-source",
+                        type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT,
+                    ),
+                ],
+            ),
+            name="Pipeline with CloudPrem Buffer",
+        ),
+        type="pipelines",
+    ),
+)
+
+configuration = Configuration()
+with ApiClient(configuration) as api_client:
+    api_instance = ObservabilityPipelinesApi(api_client)
+    response = api_instance.validate_pipeline(body=body)
+
+    print(response)

src/datadog_api_client/v2/model/observability_pipeline_amazon_s3_generic_destination.py

@@ -18,6 +18,7 @@
     from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_batch_settings import (
         ObservabilityPipelineAmazonS3GenericBatchSettings,
     )
+    from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
     from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression import (
         ObservabilityPipelineAmazonS3GenericCompression,
     )
@@ -30,6 +31,15 @@
     from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_destination_type import (
         ObservabilityPipelineAmazonS3GenericDestinationType,
     )
+    from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import (
+        ObservabilityPipelineDiskBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import (
+        ObservabilityPipelineMemoryBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import (
+        ObservabilityPipelineMemoryBufferSizeOptions,
+    )
     from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression_zstd import (
         ObservabilityPipelineAmazonS3GenericCompressionZstd,
     )
@@ -54,6 +64,7 @@ def openapi_types(_):
         from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_batch_settings import (
             ObservabilityPipelineAmazonS3GenericBatchSettings,
         )
+        from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
         from datadog_api_client.v2.model.observability_pipeline_amazon_s3_generic_compression import (
             ObservabilityPipelineAmazonS3GenericCompression,
         )
@@ -71,6 +82,7 @@ def openapi_types(_):
             "auth": (ObservabilityPipelineAwsAuth,),
             "batch_settings": (ObservabilityPipelineAmazonS3GenericBatchSettings,),
             "bucket": (str,),
+            "buffer": (ObservabilityPipelineBufferOptions,),
             "compression": (ObservabilityPipelineAmazonS3GenericCompression,),
             "encoding": (ObservabilityPipelineAmazonS3GenericEncoding,),
             "id": (str,),
@@ -85,6 +97,7 @@ def openapi_types(_):
         "auth": "auth",
         "batch_settings": "batch_settings",
         "bucket": "bucket",
+        "buffer": "buffer",
         "compression": "compression",
         "encoding": "encoding",
         "id": "id",
@@ -116,6 +129,13 @@ def __init__(
         type: ObservabilityPipelineAmazonS3GenericDestinationType,
         auth: Union[ObservabilityPipelineAwsAuth, UnsetType] = unset,
         batch_settings: Union[ObservabilityPipelineAmazonS3GenericBatchSettings, UnsetType] = unset,
+        buffer: Union[
+            ObservabilityPipelineBufferOptions,
+            ObservabilityPipelineDiskBufferOptions,
+            ObservabilityPipelineMemoryBufferOptions,
+            ObservabilityPipelineMemoryBufferSizeOptions,
+            UnsetType,
+        ] = unset,
         key_prefix: Union[str, UnsetType] = unset,
         **kwargs,
     ):
@@ -134,6 +154,9 @@ def __init__(
         :param bucket: S3 bucket name.
         :type bucket: str
 
+        :param buffer: Configuration for buffer settings on destination components.
+        :type buffer: ObservabilityPipelineBufferOptions, optional
+
         :param compression: Compression algorithm applied to encoded logs.
         :type compression: ObservabilityPipelineAmazonS3GenericCompression
 
@@ -162,6 +185,8 @@ def __init__(
             kwargs["auth"] = auth
         if batch_settings is not unset:
             kwargs["batch_settings"] = batch_settings
+        if buffer is not unset:
+            kwargs["buffer"] = buffer
         if key_prefix is not unset:
             kwargs["key_prefix"] = key_prefix
         super().__init__(kwargs)

src/datadog_api_client/v2/model/observability_pipeline_cloud_prem_destination.py

@@ -14,26 +14,39 @@
 
 
 if TYPE_CHECKING:
+    from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
     from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import (
         ObservabilityPipelineCloudPremDestinationType,
     )
+    from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import (
+        ObservabilityPipelineDiskBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import (
+        ObservabilityPipelineMemoryBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import (
+        ObservabilityPipelineMemoryBufferSizeOptions,
+    )
 
 
 class ObservabilityPipelineCloudPremDestination(ModelNormal):
     @cached_property
     def openapi_types(_):
+        from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
         from datadog_api_client.v2.model.observability_pipeline_cloud_prem_destination_type import (
             ObservabilityPipelineCloudPremDestinationType,
         )
 
         return {
+            "buffer": (ObservabilityPipelineBufferOptions,),
             "endpoint_url_key": (str,),
             "id": (str,),
             "inputs": ([str],),
             "type": (ObservabilityPipelineCloudPremDestinationType,),
         }
 
     attribute_map = {
+        "buffer": "buffer",
         "endpoint_url_key": "endpoint_url_key",
         "id": "id",
         "inputs": "inputs",
@@ -45,6 +58,13 @@ def __init__(
         id: str,
         inputs: List[str],
         type: ObservabilityPipelineCloudPremDestinationType,
+        buffer: Union[
+            ObservabilityPipelineBufferOptions,
+            ObservabilityPipelineDiskBufferOptions,
+            ObservabilityPipelineMemoryBufferOptions,
+            ObservabilityPipelineMemoryBufferSizeOptions,
+            UnsetType,
+        ] = unset,
         endpoint_url_key: Union[str, UnsetType] = unset,
         **kwargs,
     ):
@@ -53,6 +73,9 @@ def __init__(
 
         **Supported pipeline types:** logs
 
+        :param buffer: Configuration for buffer settings on destination components.
+        :type buffer: ObservabilityPipelineBufferOptions, optional
+
         :param endpoint_url_key: Name of the environment variable or secret that holds the CloudPrem endpoint URL.
         :type endpoint_url_key: str, optional
 
@@ -65,6 +88,8 @@ def __init__(
         :param type: The destination type. The value should always be ``cloud_prem``.
         :type type: ObservabilityPipelineCloudPremDestinationType
         """
+        if buffer is not unset:
+            kwargs["buffer"] = buffer
         if endpoint_url_key is not unset:
             kwargs["endpoint_url_key"] = endpoint_url_key
         super().__init__(kwargs)

src/datadog_api_client/v2/model/observability_pipeline_http_client_destination.py

@@ -17,6 +17,7 @@
     from datadog_api_client.v2.model.observability_pipeline_http_client_destination_auth_strategy import (
         ObservabilityPipelineHttpClientDestinationAuthStrategy,
     )
+    from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
     from datadog_api_client.v2.model.observability_pipeline_http_client_destination_compression import (
         ObservabilityPipelineHttpClientDestinationCompression,
     )
@@ -27,6 +28,15 @@
     from datadog_api_client.v2.model.observability_pipeline_http_client_destination_type import (
         ObservabilityPipelineHttpClientDestinationType,
     )
+    from datadog_api_client.v2.model.observability_pipeline_disk_buffer_options import (
+        ObservabilityPipelineDiskBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_options import (
+        ObservabilityPipelineMemoryBufferOptions,
+    )
+    from datadog_api_client.v2.model.observability_pipeline_memory_buffer_size_options import (
+        ObservabilityPipelineMemoryBufferSizeOptions,
+    )
 
 
 class ObservabilityPipelineHttpClientDestination(ModelNormal):
@@ -35,6 +45,7 @@ def openapi_types(_):
         from datadog_api_client.v2.model.observability_pipeline_http_client_destination_auth_strategy import (
             ObservabilityPipelineHttpClientDestinationAuthStrategy,
         )
+        from datadog_api_client.v2.model.observability_pipeline_buffer_options import ObservabilityPipelineBufferOptions
         from datadog_api_client.v2.model.observability_pipeline_http_client_destination_compression import (
             ObservabilityPipelineHttpClientDestinationCompression,
         )
@@ -48,6 +59,7 @@ def openapi_types(_):
 
         return {
             "auth_strategy": (ObservabilityPipelineHttpClientDestinationAuthStrategy,),
+            "buffer": (ObservabilityPipelineBufferOptions,),
             "compression": (ObservabilityPipelineHttpClientDestinationCompression,),
             "custom_key": (str,),
             "encoding": (ObservabilityPipelineHttpClientDestinationEncoding,),
@@ -63,6 +75,7 @@ def openapi_types(_):
 
     attribute_map = {
         "auth_strategy": "auth_strategy",
+        "buffer": "buffer",
         "compression": "compression",
         "custom_key": "custom_key",
         "encoding": "encoding",
@@ -83,6 +96,13 @@ def __init__(
         inputs: List[str],
         type: ObservabilityPipelineHttpClientDestinationType,
         auth_strategy: Union[ObservabilityPipelineHttpClientDestinationAuthStrategy, UnsetType] = unset,
+        buffer: Union[
+            ObservabilityPipelineBufferOptions,
+            ObservabilityPipelineDiskBufferOptions,
+            ObservabilityPipelineMemoryBufferOptions,
+            ObservabilityPipelineMemoryBufferSizeOptions,
+            UnsetType,
+        ] = unset,
         compression: Union[ObservabilityPipelineHttpClientDestinationCompression, UnsetType] = unset,
         custom_key: Union[str, UnsetType] = unset,
         password_key: Union[str, UnsetType] = unset,
@@ -100,6 +120,9 @@ def __init__(
         :param auth_strategy: HTTP authentication strategy.
         :type auth_strategy: ObservabilityPipelineHttpClientDestinationAuthStrategy, optional
 
+        :param buffer: Configuration for buffer settings on destination components.
+        :type buffer: ObservabilityPipelineBufferOptions, optional
+
         :param compression: Compression configuration for HTTP requests.
         :type compression: ObservabilityPipelineHttpClientDestinationCompression, optional
 
@@ -135,6 +158,8 @@ def __init__(
         """
         if auth_strategy is not unset:
             kwargs["auth_strategy"] = auth_strategy
+        if buffer is not unset:
+            kwargs["buffer"] = buffer
         if compression is not unset:
             kwargs["compression"] = compression
         if custom_key is not unset: