Add support for Spring WebClient (and thus Reactor-Netty)

Author: pimterry

README.md

@@ -17,6 +17,8 @@ Traffic can be captured from at least:
 - [x] Retrofit
 - [x] Jetty-Client v9, 10 & 11
 - [x] Async-Http-Client
+- [x] Reactor-Netty
+- [x] Spring WebClient
 
 This will also capture HTTP(S) from any downstream libraries based on each of these clients, and many other untested clients sharing similar implementations, and so should cover a very large percentage of HTTP client usage.
 

build.gradle

@@ -30,6 +30,7 @@ dependencies {
     compileOnly group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.0.3'
     compileOnly group: 'org.eclipse.jetty', name: 'jetty-client', version: '11.0.1'
     compileOnly group: 'org.asynchttpclient', name: 'async-http-client', version: '2.12.2'
+    compileOnly group: 'io.projectreactor.netty', name: 'reactor-netty', version: '1.0.4'
 
     // Test deps:
     testImplementation group: 'io.kotest', name: 'kotest-runner-junit5-jvm', version: '4.4.0'

src/main/java/tech/httptoolkit/javaagent/reactornetty/ReactorNettyResetAllConfigAdvice.java

@@ -0,0 +1,67 @@
+package tech.httptoolkit.javaagent.reactornetty;
+
+import io.netty.handler.ssl.SslContextBuilder;
+import net.bytebuddy.asm.Advice;
+import reactor.netty.http.client.HttpClientConfig;
+import reactor.netty.tcp.SslProvider;
+import reactor.netty.transport.ClientTransportConfig;
+import reactor.netty.transport.ProxyProvider;
+import tech.httptoolkit.javaagent.HttpProxyAgent;
+
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+
+public class ReactorNettyResetAllConfigAdvice {
+
+    public static final ProxyProvider agentProxyProvider = ProxyProvider.builder()
+        .type(ProxyProvider.Proxy.HTTP)
+        .address(new InetSocketAddress(
+            HttpProxyAgent.getAgentProxyHost(),
+            HttpProxyAgent.getAgentProxyPort()
+        ))
+        .build();
+
+    public static final SslProvider agentSslProvider;
+
+    public static final Field configSslField;
+    public static final Field proxyProviderField;
+
+    static {
+        System.out.println("Running netty reset static initializer");
+        try {
+            // Initialize our intercepted SSL provider:
+            agentSslProvider = SslProvider.builder()
+                .sslContext(
+                    SslContextBuilder
+                    .forClient()
+                    .trustManager(HttpProxyAgent.getInterceptedTrustManagerFactory())
+                    .build()
+                ).build();
+
+            // Rewrite the fields we want to mess with in the client config:
+            configSslField = HttpClientConfig.class.getDeclaredField("sslProvider");
+            configSslField.setAccessible(true);
+
+            proxyProviderField = ClientTransportConfig.class.getDeclaredField("proxyProvider");
+            proxyProviderField.setAccessible(true);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // Netty's HTTP Client works by creating a new client subclass, and passing the existing client's
+    // config. We hook that here: we rewrite the config whenever it's used, affecting all clients
+    // involved, and in practice anybody using config anywhere.
+
+    @Advice.OnMethodEnter
+    public static void beforeConstructor(
+        @Advice.Argument(value=0) HttpClientConfig baseHttpConfig
+    ) {
+        try {
+            configSslField.set(baseHttpConfig, agentSslProvider);
+            proxyProviderField.set(baseHttpConfig, agentProxyProvider);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/main/kotlin/tech/httptoolkit/javaagent/AgentMain.kt

@@ -81,7 +81,8 @@ fun interceptAllHttps(config: Config, instrumentation: Instrumentation) {
         JavaClientTransformer(),
         JettyClientTransformer(),
         AsyncHttpClientConfigTransformer(),
-        AsyncHttpChannelManagerTransformer()
+        AsyncHttpChannelManagerTransformer(),
+        ReactorNettyClientConfigTransformer()
     ).forEach { matchingAgentTransformer ->
         agentBuilder = matchingAgentTransformer.register(agentBuilder)
     }

src/main/kotlin/tech/httptoolkit/javaagent/ReactorNettyTransformers.kt

@@ -0,0 +1,39 @@
+package tech.httptoolkit.javaagent
+
+import net.bytebuddy.agent.builder.AgentBuilder
+import net.bytebuddy.asm.Advice
+import net.bytebuddy.description.method.MethodDescription
+import net.bytebuddy.description.type.TypeDescription
+import net.bytebuddy.dynamic.DynamicType
+import net.bytebuddy.matcher.ElementMatchers.*
+import net.bytebuddy.utility.JavaModule
+import reactor.netty.http.client.HttpClientConfig
+import tech.httptoolkit.javaagent.reactornetty.ReactorNettyResetAllConfigAdvice
+
+// To patch Reactor-Netty's HTTP client, we hook the constructor of the client itself. It has a constructor
+// that receives the config as part of every single HTTP request - we hook that to reset the relevant
+// config props every time they're used.
+
+class ReactorNettyClientConfigTransformer : MatchingAgentTransformer {
+    override fun register(builder: AgentBuilder): AgentBuilder {
+        return builder
+            .type(
+                hasSuperType(named("reactor.netty.http.client.HttpClient"))
+            ).and(
+                not(isInterface())
+            ).transform(this)
+    }
+
+    override fun transform(
+        builder: DynamicType.Builder<*>,
+        typeDescription: TypeDescription,
+        classLoader: ClassLoader?,
+        module: JavaModule?
+    ): DynamicType.Builder<*> {
+        return builder
+            .visit(Advice.to(ReactorNettyResetAllConfigAdvice::class.java)
+                .on(isConstructor<MethodDescription>().and(
+                    takesArguments(HttpClientConfig::class.java)
+                )))
+    }
+}

test-app/build.gradle

@@ -18,6 +18,8 @@ dependencies {
     implementation group: 'com.squareup.retrofit2', name: 'retrofit', version: '2.9.0'
     implementation group: 'org.eclipse.jetty', name: 'jetty-client', version: '10.0.0'
     implementation group: 'org.asynchttpclient', name: 'async-http-client', version: '2.12.2'
+    implementation group: 'org.springframework', name: 'spring-webflux', version: '5.3.4'
+    implementation group: 'io.projectreactor.netty', name: 'reactor-netty', version: '1.0.4'
 }
 
 test {

test-app/src/main/java/tech/httptoolkit/testapp/Main.java

@@ -19,15 +19,16 @@ public class Main {
             "okhttp-v4", new OkHttpV4Case(),
             "retrofit", new RetrofitCase(),
             "jetty-client", new JettyClientCase(),
-            "async-http-client", new AsyncHttpClientCase()
+            "async-http-client", new AsyncHttpClientCase(),
+            "spring-web", new SpringWebClientCase()
     );
 
     public static void main(String[] args) throws Exception {
         String runtimeName = ManagementFactory.getRuntimeMXBean().getName();
         String pid = runtimeName.split("@")[0];
         System.out.println("PID: " + pid); // Purely for convenient manual attachment to this process
 
-        String url = "https://example.test"; // Invalid URL: this should always fail to resolve
+        String url = "https://httpbin.org/404/"; // Always returns a 404, quelle surprise
 
         while (true) {
             AtomicBoolean allSuccessful = new AtomicBoolean(true);

test-app/src/main/java/tech/httptoolkit/testapp/cases/ClientCase.java

@@ -1,15 +1,9 @@
 package tech.httptoolkit.testapp.cases;
 
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
-
 public abstract class ClientCase<T> {
 
     public abstract T newClient(String url) throws Exception;
-    public abstract int test(String url, T client) throws IOException, InterruptedException, ExecutionException, TimeoutException;
+    public abstract int test(String url, T client) throws Exception;
 
     private String existingClientUrl;
     private T client;

test-app/src/main/java/tech/httptoolkit/testapp/cases/SpringWebClientCase.java

@@ -0,0 +1,26 @@
+package tech.httptoolkit.testapp.cases;
+
+import org.springframework.http.MediaType;
+import org.springframework.web.reactive.function.client.WebClient;
+import reactor.core.publisher.Mono;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class SpringWebClientCase extends ClientCase<WebClient> {
+    @Override
+    public WebClient newClient(String url) throws Exception {
+        return WebClient.create();
+    }
+
+    @Override
+    public int test(String url, WebClient client) throws URISyntaxException {
+        Mono<Integer> result = client.get()
+                .uri(new URI(url))
+                .accept(MediaType.ALL)
+                .exchangeToMono(response -> Mono.just(response.rawStatusCode()));
+
+        //noinspection ConstantConditions
+        return result.block();
+    }
+}