Add system tests for distributed tracing

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/main/java/io/sentry/samples/spring/boot/jakarta/DistributedTracingController.java

@@ -0,0 +1,51 @@
+package io.sentry.samples.spring.boot.jakarta;
+
+import io.opentelemetry.instrumentation.annotations.WithSpan;
+import java.nio.charset.Charset;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpHeaders;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.client.RestClient;
+
+@RestController
+@RequestMapping("/tracing/")
+public class DistributedTracingController {
+  private static final Logger LOGGER = LoggerFactory.getLogger(DistributedTracingController.class);
+  private final RestClient restClient;
+
+  public DistributedTracingController(RestClient restClient) {
+    this.restClient = restClient;
+  }
+
+  @GetMapping("{id}")
+  @WithSpan("tracingSpanThroughOtelAnnotation")
+  Person person(@PathVariable Long id) {
+    return restClient
+        .get()
+        .uri("http://localhost:8080/person/{id}", id)
+        .header(
+            HttpHeaders.AUTHORIZATION,
+            "Basic " + HttpHeaders.encodeBasicAuth("user", "password", Charset.defaultCharset()))
+        .retrieve()
+        .body(Person.class);
+  }
+
+  @PostMapping
+  Person create(@RequestBody Person person) {
+    return restClient
+        .post()
+        .uri("http://localhost:8080/person/")
+        .body(person)
+        .header(
+            HttpHeaders.AUTHORIZATION,
+            "Basic " + HttpHeaders.encodeBasicAuth("user", "password", Charset.defaultCharset()))
+        .retrieve()
+        .body(Person.class);
+  }
+}

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/DistributedTracingSystemTest.kt

@@ -0,0 +1,182 @@
+package io.sentry.systemtest
+
+import io.sentry.protocol.SentryId
+import io.sentry.samples.spring.boot.jakarta.Person
+import io.sentry.systemtest.util.TestHelper
+import org.junit.Before
+import org.springframework.http.HttpStatus
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
+
+class DistributedTracingSystemTest {
+
+    lateinit var testHelper: TestHelper
+
+    @Before
+    fun setup() {
+        testHelper = TestHelper("http://localhost:8080")
+        testHelper.reset()
+    }
+
+    @Test
+    fun `get person distributed tracing`() {
+        val traceId = SentryId()
+        val restClient = testHelper.restClient
+        restClient.getPersonDistributedTracing(
+            1L,
+            "$traceId-424cffc8f94feeee-1",
+            "sentry-public_key=502f25099c204a2fbf4cb16edc5975d1,sentry-sample_rand=0.456789,sentry-sample_rate=0.5,sentry-sampled=true,sentry-trace_id=$traceId,sentry-transaction=HTTP%20GET"
+        )
+        assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, restClient.lastKnownStatusCode)
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "GET /tracing/{id}" &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+        }
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "GET /person/{id}" &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+        }
+    }
+
+    @Test
+    fun `get person distributed tracing with sampled false`() {
+        val traceId = SentryId()
+        val restClient = testHelper.restClient
+        restClient.getPersonDistributedTracing(
+            1L,
+            "$traceId-424cffc8f94feeee-0",
+            "sentry-public_key=502f25099c204a2fbf4cb16edc5975d1,sentry-sample_rand=0.456789,sentry-sample_rate=0.5,sentry-sampled=false,sentry-trace_id=$traceId,sentry-transaction=HTTP%20GET"
+        )
+        assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, restClient.lastKnownStatusCode)
+
+        testHelper.ensureNoTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "GET /tracing/{id}"
+        }
+
+        testHelper.ensureNoTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "GET /person/{id}"
+        }
+    }
+
+    @Test
+    fun `get person distributed tracing without sample_rand`() {
+        val traceId = SentryId()
+        val restClient = testHelper.restClient
+        restClient.getPersonDistributedTracing(
+            1L,
+            "$traceId-424cffc8f94feeee-1",
+            "sentry-public_key=502f25099c204a2fbf4cb16edc5975d1,sentry-sample_rate=0.5,sentry-sampled=true,sentry-trace_id=$traceId,sentry-transaction=HTTP%20GET"
+        )
+        assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, restClient.lastKnownStatusCode)
+
+        var sampleRand1: String? = null
+        var sampleRand2: String? = null
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+
+            val matches = transaction.transaction == "GET /tracing/{id}" &&
+                envelopeHeader.traceContext!!.traceId == traceId &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+
+            if (matches) {
+                testHelper.logObject(envelopeHeader)
+                testHelper.logObject(transaction)
+                sampleRand1 = envelopeHeader.traceContext?.sampleRand
+            }
+
+            matches
+        }
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            val matches = transaction.transaction == "GET /person/{id}" &&
+                envelopeHeader.traceContext!!.traceId == traceId &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+
+            if (matches) {
+                testHelper.logObject(envelopeHeader)
+                testHelper.logObject(transaction)
+                sampleRand2 = envelopeHeader.traceContext?.sampleRand
+            }
+
+            matches
+        }
+
+        assertEquals(sampleRand1, sampleRand2)
+    }
+
+    @Test
+    fun `get person distributed tracing updates sample_rate on deferred decision`() {
+        val traceId = SentryId()
+        val restClient = testHelper.restClient
+        restClient.getPersonDistributedTracing(
+            1L,
+            "$traceId-424cffc8f94feeee",
+            "sentry-public_key=502f25099c204a2fbf4cb16edc5975d1,sentry-sample_rate=0.5,sentry-trace_id=$traceId,sentry-transaction=HTTP%20GET"
+        )
+        assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, restClient.lastKnownStatusCode)
+
+        var sampleRate1: String? = null
+        var sampleRate2: String? = null
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+
+            val matches = transaction.transaction == "GET /tracing/{id}" &&
+                envelopeHeader.traceContext!!.traceId == traceId &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+
+            if (matches) {
+                testHelper.logObject(envelopeHeader)
+                testHelper.logObject(transaction)
+                sampleRate1 = envelopeHeader.traceContext?.sampleRate
+            }
+
+            matches
+        }
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            val matches = transaction.transaction == "GET /person/{id}" &&
+                envelopeHeader.traceContext!!.traceId == traceId &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+
+            if (matches) {
+                testHelper.logObject(envelopeHeader)
+                testHelper.logObject(transaction)
+                sampleRate2 = envelopeHeader.traceContext?.sampleRate
+            }
+
+            matches
+        }
+
+        assertEquals(sampleRate1, sampleRate2)
+        assertNotEquals(sampleRate1, "0.5")
+    }
+
+    @Test
+    fun `create person distributed tracing`() {
+        val traceId = SentryId()
+        val restClient = testHelper.restClient
+        val person = Person("firstA", "lastB")
+        val returnedPerson = restClient.createPersonDistributedTracing(
+            person,
+            "$traceId-424cffc8f94feeee-1",
+            "sentry-public_key=502f25099c204a2fbf4cb16edc5975d1,sentry-sample_rand=0.456789,sentry-sample_rate=0.5,sentry-sampled=true,sentry-trace_id=$traceId,sentry-transaction=HTTP%20GET"
+        )
+        assertEquals(HttpStatus.OK, restClient.lastKnownStatusCode)
+
+        assertEquals(person.firstName, returnedPerson!!.firstName)
+        assertEquals(person.lastName, returnedPerson!!.lastName)
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "POST /tracing/" &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+        }
+
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
+            transaction.transaction == "POST /person/" &&
+                testHelper.doesTransactionHaveTraceId(transaction, traceId.toString())
+        }
+    }
+}

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/GraphqlGreetingSystemTest.kt

@@ -19,7 +19,7 @@ class GraphqlGreetingSystemTest {
         val response = testHelper.graphqlClient.greet("world")
 
         testHelper.ensureNoErrors(response)
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Query.greeting")
         }
     }
@@ -32,7 +32,7 @@ class GraphqlGreetingSystemTest {
         testHelper.ensureErrorReceived { error ->
             error.message?.message?.startsWith("Unresolved RuntimeException for executionId ") ?: false
         }
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Query.greeting")
         }
     }

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/GraphqlProjectSystemTest.kt

@@ -23,7 +23,7 @@ class GraphqlProjectSystemTest {
 
         testHelper.ensureNoErrors(response)
         assertEquals("proj-slug", response?.data?.project?.slug)
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Query.project")
         }
     }
@@ -34,7 +34,7 @@ class GraphqlProjectSystemTest {
 
         testHelper.ensureNoErrors(response)
         assertNotNull(response?.data?.addProject)
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Mutation.addProject")
         }
     }
@@ -48,7 +48,7 @@ class GraphqlProjectSystemTest {
         testHelper.ensureErrorReceived { error ->
             error.message?.message?.startsWith("Unresolved RuntimeException for executionId ") ?: false
         }
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Mutation.addProject")
         }
     }

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/GraphqlTaskSystemTest.kt

@@ -30,7 +30,7 @@ class GraphqlTaskSystemTest {
         assertEquals("C3", firstTask.creatorId)
         assertEquals("C3", firstTask.creator?.id)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithDescription(transaction, "Query.tasks")
         }
     }

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/PersonSystemTest.kt

@@ -23,7 +23,7 @@ class PersonSystemTest {
         restClient.getPerson(1L)
         assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, restClient.lastKnownStatusCode)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithOp(transaction, "spanCreatedThroughOtelApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "spanCreatedThroughSentryApi")
         }
@@ -39,7 +39,7 @@ class PersonSystemTest {
         assertEquals(person.firstName, returnedPerson!!.firstName)
         assertEquals(person.lastName, returnedPerson!!.lastName)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithOp(transaction, "spanCreatedThroughOtelApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "spanCreatedThroughSentryApi")
         }

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/TodoSystemTest.kt

@@ -22,7 +22,7 @@ class TodoSystemTest {
         restClient.getTodo(1L)
         assertEquals(HttpStatus.OK, restClient.lastKnownStatusCode)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithOp(transaction, "todoSpanOtelApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "todoSpanSentryApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "http.client")
@@ -35,7 +35,7 @@ class TodoSystemTest {
         restClient.getTodoWebclient(1L)
         assertEquals(HttpStatus.OK, restClient.lastKnownStatusCode)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithOp(transaction, "http.client")
         }
     }
@@ -46,7 +46,7 @@ class TodoSystemTest {
         restClient.getTodoRestClient(1L)
         assertEquals(HttpStatus.OK, restClient.lastKnownStatusCode)
 
-        testHelper.ensureTransactionReceived { transaction ->
+        testHelper.ensureTransactionReceived { transaction, envelopeHeader ->
             testHelper.doesTransactionContainSpanWithOp(transaction, "todoRestClientSpanOtelApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "todoRestClientSpanSentryApi") &&
                 testHelper.doesTransactionContainSpanWithOp(transaction, "http.client")

sentry-samples/sentry-samples-spring-boot-jakarta-opentelemetry-noagent/src/test/kotlin/io/sentry/systemtest/util/RestTestClient.kt

@@ -33,6 +33,36 @@ class RestTestClient(private val backendBaseUrl: String) : LoggingInsecureRestCl
         }
     }
 
+    fun getPersonDistributedTracing(id: Long, sentryTraceHeader: String? = null, baggageHeader: String? = null): Person? {
+        return try {
+            val response = restTemplate().exchange("$backendBaseUrl/tracing/{id}", HttpMethod.GET, entityWithAuth(headerCallback = tracingHeaders(sentryTraceHeader, baggageHeader)), Person::class.java, mapOf("id" to id))
+            lastKnownStatusCode = response.statusCode
+            response.body
+        } catch (e: HttpStatusCodeException) {
+            lastKnownStatusCode = e.statusCode
+            null
+        }
+    }
+
+    fun createPersonDistributedTracing(person: Person, sentryTraceHeader: String? = null, baggageHeader: String? = null): Person? {
+        return try {
+            val response = restTemplate().exchange("$backendBaseUrl/tracing/", HttpMethod.POST, entityWithAuth(person, tracingHeaders(sentryTraceHeader, baggageHeader)), Person::class.java, person)
+            lastKnownStatusCode = response.statusCode
+            response.body
+        } catch (e: HttpStatusCodeException) {
+            lastKnownStatusCode = e.statusCode
+            null
+        }
+    }
+
+    private fun tracingHeaders(sentryTraceHeader: String?, baggageHeader: String?): (HttpHeaders) -> HttpHeaders {
+        return { httpHeaders ->
+            sentryTraceHeader?.let { httpHeaders.set("sentry-trace", it) }
+            baggageHeader?.let { httpHeaders.set("baggage", it) }
+            httpHeaders
+        }
+    }
+
     fun getTodo(id: Long): Todo? {
         return try {
             val response = restTemplate().exchange("$backendBaseUrl/todo/{id}", HttpMethod.GET, entityWithAuth(), Todo::class.java, mapOf("id" to id))
@@ -66,11 +96,13 @@ class RestTestClient(private val backendBaseUrl: String) : LoggingInsecureRestCl
         }
     }
 
-    private fun entityWithAuth(request: Any? = null): HttpEntity<Any?> {
+    private fun entityWithAuth(request: Any? = null, headerCallback: ((HttpHeaders) -> HttpHeaders)? = null): HttpEntity<Any?> {
         val headers = HttpHeaders().also {
             it.setBasicAuth("user", "password")
         }
 
-        return HttpEntity<Any?>(request, headers)
+        val modifiedHeaders = headerCallback?.invoke(headers) ?: headers
+
+        return HttpEntity<Any?>(request, modifiedHeaders)
     }
 }