Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 50 additions & 55 deletions src/sinks/axiom/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,48 +23,50 @@ use crate::{

static CLOUD_URL: &str = "https://api.axiom.co";

/// Configuration of the URL/region to use when interacting with Axiom.
/// Axiom edge deployment configuration for data ingestion.
#[configurable_component]
#[derive(Clone, Debug, Default)]
#[serde(default)]
pub struct UrlOrRegion {
/// URI of the Axiom endpoint to send data to.
/// Full URI of the Axiom ingest endpoint.
///
/// If a path is provided, the URL is used as-is.
/// If no path (or only `/`) is provided, `/v1/datasets/{dataset}/ingest` is appended for backwards compatibility.
/// This takes precedence over `region` if both are set (but both should not be set).
/// Use this for the legacy EU instance, local development, or custom endpoints.
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you mean to include this?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pront Yes, this is customer-facing.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I find it more confusing that what we had before. Is it implying setting this to None for all other cases?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
/// Use this for the legacy EU instance, local development, or custom endpoints.
/// Use `url` for the legacy EU instance, local development, or custom endpoints.
/// In all other cases, use `region`.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pront Does it make it clearer?

/// If the path is empty or `/`, Vector appends `/v1/datasets/{dataset}/ingest`
/// for backwards compatibility with the legacy endpoint format.
/// Don't set both `url` and `region`.
/// `url` takes precedence over `region` if you set both.
#[configurable(validation(format = "uri"))]
#[configurable(metadata(docs::examples = "https://api.eu.axiom.co"))]
#[configurable(metadata(docs::examples = "http://localhost:3400/ingest"))]
#[configurable(metadata(docs::examples = "${AXIOM_URL}"))]
pub url: Option<String>,

/// The Axiom regional edge domain to use for ingestion.
/// Domain of the Axiom edge deployment.
///
/// Specify the domain name only (no scheme, no path).
/// When set, data is sent to `https://{region}/v1/ingest/{dataset}`.
/// Cannot be used together with `url`.
#[configurable(metadata(docs::examples = "${AXIOM_REGION}"))]
#[configurable(metadata(docs::examples = "mumbai.axiom.co"))]
/// Specify the domain name only without the scheme or path.
/// Vector sends data to `https://{region}/v1/ingest/{dataset}`.
/// Don’t set both `url` and `region`.
#[configurable(metadata(docs::examples = "${AXIOM_DOMAIN}"))]
#[configurable(metadata(docs::examples = "eu-central-1.aws.edge.axiom.co"))]
#[configurable(metadata(docs::examples = "us-east-1.aws.edge.axiom.co"))]
pub region: Option<String>,
}

impl UrlOrRegion {
/// Validates that url and region are not both set.
/// Validates that `url` and `region` are mutually exclusive.
fn validate(&self) -> crate::Result<()> {
if self.url.is_some() && self.region.is_some() {
return Err("Cannot set both `url` and `region`. Please use only one.".into());
}
Ok(())
}

/// Returns the url if set.
/// Returns the URL if set.
pub fn url(&self) -> Option<&str> {
self.url.as_deref()
}

/// Returns the region if set.
/// Returns the edge deployment domain if set.
pub fn region(&self) -> Option<&str> {
self.region.as_deref()
}
Expand All @@ -74,24 +76,24 @@ impl UrlOrRegion {
#[configurable_component(sink("axiom", "Deliver log events to Axiom."))]
#[derive(Clone, Debug, Default)]
pub struct AxiomConfig {
/// The Axiom organization ID.
/// Axiom organization ID.
///
/// Only required when using personal tokens.
#[configurable(metadata(docs::examples = "${AXIOM_ORG_ID}"))]
#[configurable(metadata(docs::examples = "123abc"))]
pub org_id: Option<String>,

/// The Axiom API token.
/// Axiom API token.
#[configurable(metadata(docs::examples = "${AXIOM_TOKEN}"))]
#[configurable(metadata(docs::examples = "123abc"))]
pub token: SensitiveString,

/// The Axiom dataset to write to.
/// Axiom dataset to write to.
#[configurable(metadata(docs::examples = "${AXIOM_DATASET}"))]
#[configurable(metadata(docs::examples = "vector_rocks"))]
pub dataset: String,

/// Configuration for the URL or regional edge endpoint.
/// URL or edge deployment endpoint configuration.
#[serde(flatten)]
#[configurable(derived)]
pub endpoint: UrlOrRegion,
Expand All @@ -100,23 +102,21 @@ pub struct AxiomConfig {
#[serde(default)]
pub request: RequestConfig,

/// The compression algorithm to use.
/// Compression algorithm.
#[configurable(derived)]
#[serde(default = "Compression::zstd_default")]
pub compression: Compression,

/// The TLS settings for the connection.
///
/// Optional, constrains TLS settings for this sink.
/// TLS settings for the connection.
#[configurable(derived)]
pub tls: Option<TlsConfig>,

/// The batch settings for the sink.
/// Batch settings.
#[configurable(derived)]
#[serde(default)]
pub batch: BatchConfig<RealtimeSizeBasedDefaultBatchSettings>,

/// Controls how acknowledgements are handled for this sink.
/// Acknowledgement settings.
#[configurable(derived)]
#[serde(
default,
Expand All @@ -142,23 +142,19 @@ impl GenerateConfig for AxiomConfig {
#[typetag::serde(name = "axiom")]
impl SinkConfig for AxiomConfig {
async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> {
// Validate that url and region are not both set
// Validate that `url` and `region` are mutually exclusive.
self.endpoint.validate()?;

let mut request = self.request.clone();
if let Some(org_id) = &self.org_id {
// NOTE: Only add the org id header if an org id is provided
// Add the org ID header for personal token authentication.
request
.headers
.insert("X-Axiom-Org-Id".to_string(), org_id.clone());
}

// Axiom has a custom high-performance database that can be ingested
// into using the native HTTP ingest endpoint. This configuration wraps
// the vector HTTP sink with the necessary adjustments to send data
// to Axiom, whilst keeping the configuration simple and easy to use
// and maintenance of the vector axiom sink to a minimum.
//
// This sink wraps Vector's HTTP sink with Axiom-specific defaults
// for simple configuration and minimal maintenance.
let http_sink_config = HttpSinkConfig {
uri: self.build_endpoint().try_into()?,
compression: self.compression,
Expand All @@ -178,8 +174,8 @@ impl SinkConfig for AxiomConfig {
}),
Transformer::default(),
),
payload_prefix: "".into(), // Always newline delimited JSON
payload_suffix: "".into(), // Always newline delimited JSON
payload_prefix: "".into(), // Newline-delimited JSON
payload_suffix: "".into(), // Newline-delimited JSON
};

http_sink_config.build(cx).await
Expand All @@ -195,35 +191,34 @@ impl SinkConfig for AxiomConfig {
}

impl AxiomConfig {
/// Builds the ingest endpoint URL.
///
/// Priority: `url` > `region` > default cloud endpoint.
fn build_endpoint(&self) -> String {
// Priority: url > region > default cloud endpoint

// If url is set, check if it has a path
// Use explicit URL if set.
if let Some(url) = self.endpoint.url() {
let url = url.trim_end_matches('/');

// Parse URL to check if path is provided
// If path is empty or just "/", append the legacy format for backwards compatibility
// Otherwise, use the URL as-is
// If the path is empty, append the legacy path format for backwards
// compatibility with the legacy EU instance and older configurations.
if let Ok(parsed) = url::Url::parse(url) {
let path = parsed.path();
if path.is_empty() || path == "/" {
// Backwards compatibility: append legacy path format
return format!("{url}/v1/datasets/{}/ingest", self.dataset);
}
}

// URL has a custom path, use as-is
// Custom path provided, use as-is.
return url.to_string();
}

// If region is set, build the regional edge endpoint
// Use edge deployment domain if set.
if let Some(region) = self.endpoint.region() {
let region = region.trim_end_matches('/');
return format!("https://{region}/v1/ingest/{}", self.dataset);
}

// Default: use cloud endpoint with legacy path format
// Default to US East 1 cloud endpoint with legacy path format.
format!("{CLOUD_URL}/v1/datasets/{}/ingest", self.dataset)
}
}
Expand All @@ -237,7 +232,7 @@ mod test {

#[test]
fn test_region_domain_only() {
// region: mumbai.axiomdomain.co → https://mumbai.axiomdomain.co/v1/ingest/test-3
// Edge deployment builds the correct ingest URL.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
region: Some("mumbai.axiomdomain.co".to_string()),
Expand All @@ -252,7 +247,7 @@ mod test {

#[test]
fn test_default_no_config() {
// No url, no region → https://api.axiom.co/v1/datasets/foo/ingest
// Without URL or edge deployment, defaults to `https://api.axiom.co/v1/datasets/DATASET_NAME/ingest`.
let config = super::AxiomConfig {
dataset: "foo".to_string(),
..Default::default()
Expand All @@ -263,7 +258,7 @@ mod test {

#[test]
fn test_url_with_custom_path() {
// url: http://localhost:3400/ingest → http://localhost:3400/ingest (as-is)
// Custom path in URL is used as-is (local development).
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
url: Some("http://localhost:3400/ingest".to_string()),
Expand All @@ -278,7 +273,7 @@ mod test {

#[test]
fn test_url_without_path_backwards_compat() {
// url: https://api.eu.axiom.co/ → https://api.eu.axiom.co/v1/datasets/qoo/ingest
// Legacy EU instance URL appends the legacy path format.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
url: Some("https://api.eu.axiom.co".to_string()),
Expand All @@ -290,7 +285,7 @@ mod test {
let endpoint = config.build_endpoint();
assert_eq!(endpoint, "https://api.eu.axiom.co/v1/datasets/qoo/ingest");

// Also test with trailing slash
// Trailing slash is handled correctly.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
url: Some("https://api.eu.axiom.co/".to_string()),
Expand All @@ -305,7 +300,7 @@ mod test {

#[test]
fn test_both_url_and_region_fails_validation() {
// When both url and region are set, validation should fail
// Setting both URL and edge deployment fails validation.
let endpoint = super::UrlOrRegion {
url: Some("http://localhost:3400/ingest".to_string()),
region: Some("mumbai.axiomdomain.co".to_string()),
Expand All @@ -321,7 +316,7 @@ mod test {

#[test]
fn test_url_or_region_deserialization_with_url() {
// Test that url can be deserialized at the top level (flattened)
// URL deserializes at the top level (flattened).
let config: super::AxiomConfig = toml::from_str(
r#"
token = "test-token"
Expand All @@ -337,7 +332,7 @@ mod test {

#[test]
fn test_url_or_region_deserialization_with_region() {
// Test that region can be deserialized at the top level (flattened)
// Edge deployment deserializes at the top level (flattened).
let config: super::AxiomConfig = toml::from_str(
r#"
token = "test-token"
Expand All @@ -353,7 +348,7 @@ mod test {

#[test]
fn test_production_regional_edges() {
// Production AWS edge
// US East 1 edge deployment.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
region: Some("eu-central-1.aws.edge.axiom.co".to_string()),
Expand All @@ -371,7 +366,7 @@ mod test {

#[test]
fn test_staging_environment_edges() {
// Staging environment edge
// Staging environment edge deployment.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
region: Some("us-east-1.edge.staging.axiomdomain.co".to_string()),
Expand All @@ -389,7 +384,7 @@ mod test {

#[test]
fn test_dev_environment_edges() {
// Dev environment edge
// Dev environment edge deployment.
let config = super::AxiomConfig {
endpoint: super::UrlOrRegion {
region: Some("eu-west-1.edge.dev.axiomdomain.co".to_string()),
Expand Down
Loading
Loading