Introduce telemetry for observability

e2e-tests/src/lib.rs

@@ -97,10 +97,24 @@ pub struct LdkServerHandle {
 	client: LdkServerClient,
 }
 
+pub struct LdkServerConfig {
+	pub metrics_auth: Option<(String, String)>,
+}
+
+impl Default for LdkServerConfig {
+	fn default() -> Self {
+		Self { metrics_auth: None }
+	}
+}
+
 impl LdkServerHandle {
 	/// Starts a new ldk-server instance against the given bitcoind.
 	/// Waits until the server is ready to accept requests.
 	pub async fn start(bitcoind: &TestBitcoind) -> Self {
+		Self::start_with_config(bitcoind, LdkServerConfig::default()).await
+	}
+
+	pub async fn start_with_config(bitcoind: &TestBitcoind, config: LdkServerConfig) -> Self {
 		#[allow(deprecated)]
 		let storage_dir = tempfile::tempdir().unwrap().into_path();
 		let rest_port = find_available_port();
@@ -111,6 +125,12 @@ impl LdkServerHandle {
 
 		let exchange_name = format!("e2e_test_exchange_{rest_port}");
 
+		let metrics_auth_config = if let Some((user, pass)) = config.metrics_auth {
+			format!("username = \"{}\"\npassword = \"{}\"", user, pass)
+		} else {
+			String::new()
+		};
+
 		let config_content = format!(
 			r#"[node]
 network = "regtest"
@@ -140,6 +160,11 @@ max_client_to_self_delay = 1024
 min_payment_size_msat = 0
 max_payment_size_msat = 1000000000
 client_trusts_lsp = true
+
+[metrics]
+enabled = true
+poll_metrics_interval = 1
+{metrics_auth_config}
 "#,
 			storage_dir = storage_dir.display(),
 		);

e2e-tests/tests/e2e.rs

@@ -12,7 +12,8 @@ use std::time::Duration;
 
 use e2e_tests::{
 	find_available_port, mine_and_sync, run_cli, run_cli_raw, setup_funded_channel,
-	wait_for_onchain_balance, LdkServerHandle, RabbitMqEventConsumer, TestBitcoind,
+	wait_for_onchain_balance, LdkServerConfig, LdkServerHandle, RabbitMqEventConsumer,
+	TestBitcoind,
 };
 use hex_conservative::{DisplayHex, FromHex};
 use ldk_node::bitcoin::hashes::{sha256, Hash};
@@ -995,3 +996,132 @@ async fn test_hodl_invoice_fail() {
 		events_a.iter().map(|e| &e.event).collect::<Vec<_>>()
 	);
 }
+
+#[tokio::test]
+async fn test_metrics_endpoint() {
+	let bitcoind = TestBitcoind::new();
+
+	// Test with metrics enabled
+	let server_a = LdkServerHandle::start(&bitcoind).await;
+	let server_b = LdkServerHandle::start(&bitcoind).await;
+
+	let client = server_a.client();
+	let metrics_result = client.get_metrics().await;
+
+	assert!(metrics_result.is_ok(), "Expected metrics to succeed when enabled");
+	let metrics = metrics_result.unwrap();
+
+	// Verify initial state
+	assert!(metrics.contains("ldk_server_total_peers_count 0"));
+	assert!(metrics.contains("ldk_server_total_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_successful_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_pending_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_failed_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_public_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_private_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_onchain_balance_sats 0"));
+	assert!(metrics.contains("ldk_server_spendable_onchain_balance_sats 0"));
+	assert!(metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0"));
+	assert!(metrics.contains("ldk_server_total_lightning_balance_sats 0"));
+
+	// Set up channel and make a payment to trigger metrics update
+	setup_funded_channel(&bitcoind, &server_a, &server_b, 100_000).await;
+
+	// Poll for channel, peer and balance metrics.
+	let timeout = Duration::from_secs(10);
+	let start = std::time::Instant::now();
+	loop {
+		let metrics = client.get_metrics().await.unwrap();
+		if metrics.contains("ldk_server_total_peers_count 1")
+			&& metrics.contains("ldk_server_total_channels_count 1")
+			&& metrics.contains("ldk_server_total_public_channels_count 1")
+			&& metrics.contains("ldk_server_total_payments_count 2")
+			&& !metrics.contains("ldk_server_total_lightning_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_spendable_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0")
+		{
+			break;
+		}
+
+		if start.elapsed() > timeout {
+			let current_metrics = client.get_metrics().await.unwrap();
+			panic!(
+				"Timed out waiting for channel, peer and balance metrics to update. Current metrics:\n{}",
+				current_metrics
+			);
+		}
+		tokio::time::sleep(Duration::from_secs(1)).await;
+	}
+
+	let invoice_resp = server_b
+		.client()
+		.bolt11_receive(Bolt11ReceiveRequest {
+			amount_msat: Some(10_000_000),
+			description: Some(Bolt11InvoiceDescription {
+				kind: Some(bolt11_invoice_description::Kind::Direct("metrics test".to_string())),
+			}),
+			expiry_secs: 3600,
+		})
+		.await
+		.unwrap();
+
+	run_cli(&server_a, &["bolt11-send", &invoice_resp.invoice]);
+
+	// Wait to receive the PaymentSuccessful event and update metrics
+	let timeout = Duration::from_secs(30);
+	let start = std::time::Instant::now();
+	loop {
+		let metrics = client.get_metrics().await.unwrap();
+		if metrics.contains("ldk_server_total_successful_payments_count 1")
+			&& !metrics.contains("ldk_server_total_lightning_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_spendable_onchain_balance_sats 0")
+			&& !metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0")
+		{
+			break;
+		}
+		if start.elapsed() > timeout {
+			panic!("Timed out waiting for payment metrics to update");
+		}
+		tokio::time::sleep(Duration::from_millis(500)).await;
+	}
+}
+
+#[tokio::test]
+async fn test_metrics_endpoint_with_auth() {
+	let bitcoind = TestBitcoind::new();
+
+	let username = "admin";
+	let password = "password123";
+
+	let config =
+		LdkServerConfig { metrics_auth: Some((username.to_string(), password.to_string())) };
+
+	let server = LdkServerHandle::start_with_config(&bitcoind, config).await;
+	let client = server.client();
+
+	// Should fail because auth is provided in the config
+	let result = client.get_metrics().await;
+	assert!(result.is_err(), "Expected failure without credentials");
+
+	// Request has the correct auth, so it should succeed
+	let result = client.get_metrics_with_auth(Some(username), Some(password)).await;
+
+	assert!(result.is_ok(), "Expected success with correct credentials");
+	let metrics = result.unwrap();
+
+	assert!(metrics.contains("ldk_server_total_peers_count 0"));
+	assert!(metrics.contains("ldk_server_total_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_successful_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_pending_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_failed_payments_count 0"));
+	assert!(metrics.contains("ldk_server_total_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_public_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_private_channels_count 0"));
+	assert!(metrics.contains("ldk_server_total_onchain_balance_sats 0"));
+	assert!(metrics.contains("ldk_server_spendable_onchain_balance_sats 0"));
+	assert!(metrics.contains("ldk_server_total_anchor_channels_reserve_sats 0"));
+	assert!(metrics.contains("ldk_server_total_lightning_balance_sats 0"));
+}

ldk-server-client/src/client.rs

@@ -40,13 +40,15 @@ use ldk_server_protos::endpoints::{
 	BOLT11_RECEIVE_VIA_JIT_CHANNEL_PATH, BOLT11_SEND_PATH, BOLT12_RECEIVE_PATH, BOLT12_SEND_PATH,
 	CLOSE_CHANNEL_PATH, CONNECT_PEER_PATH, DECODE_INVOICE_PATH, DECODE_OFFER_PATH,
 	DISCONNECT_PEER_PATH, EXPORT_PATHFINDING_SCORES_PATH, FORCE_CLOSE_CHANNEL_PATH,
-	GET_BALANCES_PATH, GET_NODE_INFO_PATH, GET_PAYMENT_DETAILS_PATH, GRAPH_GET_CHANNEL_PATH,
-	GRAPH_GET_NODE_PATH, GRAPH_LIST_CHANNELS_PATH, GRAPH_LIST_NODES_PATH, LIST_CHANNELS_PATH,
-	LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, LIST_PEERS_PATH, ONCHAIN_RECEIVE_PATH,
-	ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH, SPLICE_OUT_PATH,
-	SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH, VERIFY_SIGNATURE_PATH,
+	GET_BALANCES_PATH, GET_METRICS_PATH, GET_NODE_INFO_PATH, GET_PAYMENT_DETAILS_PATH,
+	GRAPH_GET_CHANNEL_PATH, GRAPH_GET_NODE_PATH, GRAPH_LIST_CHANNELS_PATH, GRAPH_LIST_NODES_PATH,
+	LIST_CHANNELS_PATH, LIST_FORWARDED_PAYMENTS_PATH, LIST_PAYMENTS_PATH, LIST_PEERS_PATH,
+	ONCHAIN_RECEIVE_PATH, ONCHAIN_SEND_PATH, OPEN_CHANNEL_PATH, SIGN_MESSAGE_PATH, SPLICE_IN_PATH,
+	SPLICE_OUT_PATH, SPONTANEOUS_SEND_PATH, UNIFIED_SEND_PATH, UPDATE_CHANNEL_CONFIG_PATH,
+	VERIFY_SIGNATURE_PATH,
 };
 use ldk_server_protos::error::{ErrorCode, ErrorResponse};
+use prost::bytes::Bytes;
 use prost::Message;
 use reqwest::header::CONTENT_TYPE;
 use reqwest::{Certificate, Client};
@@ -70,6 +72,11 @@ pub struct LdkServerClient {
 	api_key: String,
 }
 
+enum RequestType {
+	Get,
+	Post,
+}
+
 impl LdkServerClient {
 	/// Constructs a [`LdkServerClient`] using `base_url` as the ldk-server endpoint.
 	///
@@ -115,6 +122,27 @@ impl LdkServerClient {
 		self.post_request(&request, &url).await
 	}
 
+	/// Retrieve the node metrics in Prometheus format.
+	pub async fn get_metrics(&self) -> Result<String, LdkServerError> {
+		self.get_metrics_with_auth(None, None).await
+	}
+
+	/// Retrieve the node metrics in Prometheus format using Basic Auth.
+	pub async fn get_metrics_with_auth(
+		&self, username: Option<&str>, password: Option<&str>,
+	) -> Result<String, LdkServerError> {
+		let url = format!("https://{}/{GET_METRICS_PATH}", self.base_url);
+		let payload =
+			self.make_request(&url, RequestType::Get, None, false, username, password).await?;
+
+		String::from_utf8(payload.to_vec()).map_err(|e| {
+			LdkServerError::new(
+				InternalError,
+				format!("Failed to decode metrics response as string: {}", e),
+			)
+		})
+	}
+
 	/// Retrieves an overview of all known balances.
 	/// For API contract/usage, refer to docs for [`GetBalancesRequest`] and [`GetBalancesResponse`].
 	pub async fn get_balances(
@@ -450,31 +478,60 @@ impl LdkServerClient {
 		&self, request: &Rq, url: &str,
 	) -> Result<Rs, LdkServerError> {
 		let request_body = request.encode_to_vec();
-		let auth_header = self.compute_auth_header(&request_body);
-		let response_raw = self
-			.client
-			.post(url)
-			.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
-			.header("X-Auth", auth_header)
-			.body(request_body)
-			.send()
-			.await
-			.map_err(|e| {
-				LdkServerError::new(InternalError, format!("HTTP request failed: {}", e))
-			})?;
+		let payload =
+			self.make_request(url, RequestType::Post, Some(request_body), true, None, None).await?;
+		Rs::decode(&payload[..]).map_err(|e| {
+			LdkServerError::new(InternalError, format!("Failed to decode success response: {}", e))
+		})
+	}
+
+	async fn make_request(
+		&self, url: &str, request_type: RequestType, body: Option<Vec<u8>>,
+		hmac_authenticated: bool, metrics_username: Option<&str>, metrics_password: Option<&str>,
+	) -> Result<Bytes, LdkServerError> {
+		let builder = match request_type {
+			RequestType::Get => self.client.get(url),
+			RequestType::Post => self.client.post(url),
+		};
+
+		let builder = if hmac_authenticated {
+			let body_for_auth = body.as_deref().unwrap_or(&[]);
+			let auth_header = self.compute_auth_header(body_for_auth);
+			builder.header("X-Auth", auth_header)
+		} else {
+			builder
+		};
+
+		let builder = if let Some(body_content) = body {
+			builder.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM).body(body_content)
+		} else {
+			builder
+		};
+
+		let builder = if let (Some(username), Some(password)) = (metrics_username, metrics_password)
+		{
+			builder.basic_auth(username, Some(password))
+		} else {
+			builder
+		};
+
+		let response_raw = builder.send().await.map_err(|e| {
+			LdkServerError::new(InternalError, format!("HTTP request failed: {}", e))
+		})?;
 
+		self.handle_response(response_raw).await
+	}
+
+	async fn handle_response(
+		&self, response_raw: reqwest::Response,
+	) -> Result<Bytes, LdkServerError> {
 		let status = response_raw.status();
 		let payload = response_raw.bytes().await.map_err(|e| {
 			LdkServerError::new(InternalError, format!("Failed to read response body: {}", e))
 		})?;
 
 		if status.is_success() {
-			Ok(Rs::decode(&payload[..]).map_err(|e| {
-				LdkServerError::new(
-					InternalError,
-					format!("Failed to decode success response: {}", e),
-				)
-			})?)
+			Ok(payload)
 		} else {
 			let error_response = ErrorResponse::decode(&payload[..]).map_err(|e| {
 				LdkServerError::new(

ldk-server-protos/src/endpoints.rs

@@ -45,3 +45,4 @@ pub const GRAPH_LIST_NODES_PATH: &str = "GraphListNodes";
 pub const GRAPH_GET_NODE_PATH: &str = "GraphGetNode";
 pub const DECODE_INVOICE_PATH: &str = "DecodeInvoice";
 pub const DECODE_OFFER_PATH: &str = "DecodeOffer";
+pub const GET_METRICS_PATH: &str = "metrics";

ldk-server/ldk-server-config.toml

@@ -88,3 +88,11 @@ client_trusts_lsp = false
 ## A token we may require to be sent by the clients.
 ## If set, only requests matching this token will be accepted. (uncomment and set if required)
 # require_token = ""
+
+# Metrics settings
+[metrics]
+enabled = false
+poll_metrics_interval = 60       # The polling interval for metrics in seconds. Defaults to 60secs if unset and metrics enabled.
+# The auth details below are optional, but uncommenting the fields means enabling basic auth, so valid fields must be supplied.
+#username = ""               # The username required to access the metrics endpoint (Basic Auth).
+#password = ""               # The password required to access the metrics endpoint (Basic Auth).