feat: Replace deprecated Doppler recentLogs with Log Cache client (#1348)

See gh-1237, gh-1348

Author: jorbaum

README.md

@@ -17,6 +17,44 @@ The `cf-java-client` project is a Java language binding for interacting with a C
 ## Versions
 The Cloud Foundry Java Client has two active versions. The `5.x` line is compatible with Spring Boot `2.4.x - 2.6.x` just to manage its dependencies, while the `4.x` line uses Spring Boot `2.3.x`.
 
+## Deprecations
+
+### `DopplerClient.recentLogs()` — Recent Logs via Doppler
+
+> [!WARNING]
+> **Deprecated since cf-java-client `5.17.x`**
+>
+> The `DopplerClient.recentLogs()` endpoint (and the related `RecentLogsRequest` / `LogMessage` types from the `org.cloudfoundry.doppler` package) are **deprecated** and will be removed in a future release.
+>
+> This API relies on the [Loggregator][loggregator] Doppler/Traffic Controller endpoint `/apps/{id}/recentlogs`, which was removed in **Loggregator ≥ 107.0**.
+> The affected platform versions are:
+>
+> | Platform | Last version with Doppler recent-logs support |
+> | -------- | --------------------------------------------- |
+> | CF Deployment (CFD) | `< 24.3` |
+> | Tanzu Application Service (TAS) | `< 4.0` |
+>
+> **Migration:** Replace any call to `DopplerClient.recentLogs()` with [`LogCacheClient.read()`][log-cache-api] (available via `org.cloudfoundry.logcache.v1.LogCacheClient`).
+>
+> ```java
+> // Before (deprecated)
+> dopplerClient.recentLogs(RecentLogsRequest.builder()
+>     .applicationId(appId)
+>     .build());
+>
+> // After
+> logCacheClient.read(ReadRequest.builder()
+>     .sourceId(appId)
+>     .envelopeTypes(EnvelopeType.LOG)
+>     .build());
+> ```
+
+> [!NOTE]
+> **Operations API users:** `Applications.logs(ApplicationLogsRequest)` now uses Log Cache under the hood for recent logs (the default). No migration is needed at the Operations layer.
+
+[loggregator]: https://github.com/cloudfoundry/loggregator
+[log-cache-api]: https://github.com/cloudfoundry/log-cache
+
 ## Dependencies
 Most projects will need two dependencies; the Operations API and an implementation of the Client API.  For Maven, the dependencies would be defined like this:
 
@@ -76,6 +114,9 @@ Both the `cloudfoundry-operations` and `cloudfoundry-client` projects follow a [
 
 ### `CloudFoundryClient`, `DopplerClient`, `UaaClient` Builders
 
+> [!NOTE]
+> **`DopplerClient` — partial deprecation:** The `recentLogs()` method on `DopplerClient` is deprecated and only works against Loggregator \< 107.0 (CFD \< 24.3 / TAS \< 4.0). See the [Deprecations](#deprecations) section above for the migration path to `LogCacheClient`.
+
 The lowest-level building blocks of the API are `ConnectionContext` and `TokenProvider`.  These types are intended to be shared between instances of the clients, and come with out of the box implementations.  To instantiate them, you configure them with builders:
 
 ```java

cloudfoundry-client/src/main/java/org/cloudfoundry/doppler/DopplerClient.java

@@ -42,9 +42,15 @@ public interface DopplerClient {
     /**
      * Makes the <a href="https://github.com/cloudfoundry/loggregator/tree/develop/src/trafficcontroller#endpoints">Recent Logs</a> request
      *
+     * @deprecated Use {@link org.cloudfoundry.logcache.v1.LogCacheClient#read(org.cloudfoundry.logcache.v1.ReadRequest)} instead.
+     * Only works with {@code Loggregator < 107.0}, shipped in {@code CFD < 24.3} and {@code TAS < 4.0}.
      * @param request the Recent Logs request
-     * @return the events from the recent logs
+     * @return a flux of events from the recent logs
+     * @see <a href="https://github.com/cloudfoundry/loggregator">Loggregator</a>
+     * @see <a href="https://github.com/cloudfoundry/log-cache">Log Cache</a>
+     * @see org.cloudfoundry.logcache.v1.LogCacheClient#read(org.cloudfoundry.logcache.v1.ReadRequest)
      */
+    @Deprecated
     Flux<Envelope> recentLogs(RecentLogsRequest request);
 
     /**

cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/_DefaultCloudFoundryOperations.java

@@ -23,6 +23,7 @@
 import org.cloudfoundry.client.v3.spaces.ListSpacesRequest;
 import org.cloudfoundry.client.v3.spaces.SpaceResource;
 import org.cloudfoundry.doppler.DopplerClient;
+import org.cloudfoundry.logcache.v1.LogCacheClient;
 import org.cloudfoundry.networking.NetworkingClient;
 import org.cloudfoundry.operations.advanced.Advanced;
 import org.cloudfoundry.operations.advanced.DefaultAdvanced;
@@ -79,7 +80,7 @@ public Advanced advanced() {
     @Override
     @Value.Derived
     public Applications applications() {
-        return new DefaultApplications(getCloudFoundryClientPublisher(), getDopplerClientPublisher(), getSpaceId());
+        return new DefaultApplications(getCloudFoundryClientPublisher(), getDopplerClientPublisher(), getLogCacheClientPublisher(), getSpaceId());
     }
 
     @Override
@@ -185,6 +186,19 @@ Mono<DopplerClient> getDopplerClientPublisher() {
             .orElse(Mono.error(new IllegalStateException("DopplerClient must be set")));
     }
 
+    /**
+     * The {@link LogCacheClient} to use for operations functionality
+     */
+    @Nullable
+    abstract LogCacheClient getLogCacheClient();
+
+    @Value.Derived
+    Mono<LogCacheClient> getLogCacheClientPublisher() {
+        return Optional.ofNullable(getLogCacheClient())
+            .map(Mono::just)
+            .orElse(Mono.error(new IllegalStateException("LogCacheClient must be set")));
+    }
+
     /**
      * The {@link NetworkingClient} to use for operations functionality
      */

cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/Applications.java

@@ -128,8 +128,9 @@ public interface Applications {
 
     /**
      * List the applications logs.
-     * Only works with {@code Loggregator < 107.0}, shipped in {@code CFD < 24.3}
-     * and {@code TAS < 4.0}.
+     * Uses Log Cache under the hood when {@link ApplicationLogsRequest#getRecent()} is {@code true}.
+     * Log streaming still uses Doppler, which is not available in CF deployments following
+     * <a href="https://docs.cloudfoundry.org/loggregator/architecture.html#shared-nothing-architecture">shared-nothing architecture</a>.
      *
      * @param request the application logs request
      * @return the applications logs

cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/DefaultApplications.java

@@ -154,6 +154,9 @@
 import org.cloudfoundry.doppler.LogMessage;
 import org.cloudfoundry.doppler.RecentLogsRequest;
 import org.cloudfoundry.doppler.StreamRequest;
+import org.cloudfoundry.logcache.v1.EnvelopeBatch;
+import org.cloudfoundry.logcache.v1.LogCacheClient;
+import org.cloudfoundry.logcache.v1.ReadRequest;
 import org.cloudfoundry.operations.util.OperationsLogging;
 import org.cloudfoundry.util.DateUtils;
 import org.cloudfoundry.util.DelayTimeoutException;
@@ -200,6 +203,10 @@ public final class DefaultApplications implements Applications {
     private static final Comparator<LogMessage> LOG_MESSAGE_COMPARATOR =
             Comparator.comparing(LogMessage::getTimestamp);
 
+    private static final Comparator<org.cloudfoundry.logcache.v1.Envelope>
+            LOG_MESSAGE_COMPARATOR_LOG_CACHE =
+                    Comparator.comparing(org.cloudfoundry.logcache.v1.Envelope::getTimestamp);
+
     private static final Duration LOG_MESSAGE_TIMESPAN = Duration.ofMillis(500);
 
     private static final int MAX_NUMBER_OF_RECENT_EVENTS = 50;
@@ -214,24 +221,29 @@ public final class DefaultApplications implements Applications {
 
     private final Mono<DopplerClient> dopplerClient;
 
+    private final Mono<LogCacheClient> logCacheClient;
+
     private final RandomWords randomWords;
 
     private final Mono<String> spaceId;
 
     public DefaultApplications(
             Mono<CloudFoundryClient> cloudFoundryClient,
             Mono<DopplerClient> dopplerClient,
+            Mono<LogCacheClient> logCacheClient,
             Mono<String> spaceId) {
-        this(cloudFoundryClient, dopplerClient, new WordListRandomWords(), spaceId);
+        this(cloudFoundryClient, dopplerClient, logCacheClient, new WordListRandomWords(), spaceId);
     }
 
     DefaultApplications(
             Mono<CloudFoundryClient> cloudFoundryClient,
             Mono<DopplerClient> dopplerClient,
+            Mono<LogCacheClient> logCacheClient,
             RandomWords randomWords,
             Mono<String> spaceId) {
         this.cloudFoundryClient = cloudFoundryClient;
         this.dopplerClient = dopplerClient;
+        this.logCacheClient = logCacheClient;
         this.randomWords = randomWords;
         this.spaceId = spaceId;
     }
@@ -529,6 +541,7 @@ public Flux<Task> listTasks(ListApplicationTasksRequest request) {
                 .checkpoint();
     }
 
+    @Deprecated
     @Override
     public Flux<LogMessage> logs(LogsRequest request) {
         return Mono.zip(this.cloudFoundryClient, this.spaceId)
@@ -546,22 +559,23 @@ public Flux<LogMessage> logs(LogsRequest request) {
 
     @Override
     public Flux<ApplicationLog> logs(ApplicationLogsRequest request) {
-        return logs(LogsRequest.builder()
-                        .name(request.getName())
-                        .recent(request.getRecent())
-                        .build())
-                .map(
-                        logMessage ->
-                                ApplicationLog.builder()
-                                        .sourceId(logMessage.getApplicationId())
-                                        .sourceType(logMessage.getSourceType())
-                                        .instanceId(logMessage.getSourceInstance())
-                                        .message(logMessage.getMessage())
-                                        .timestamp(logMessage.getTimestamp())
-                                        .logType(
-                                                ApplicationLogType.from(
-                                                        logMessage.getMessageType().name()))
-                                        .build());
+        if (request.getRecent() == null || request.getRecent()) {
+            return Mono.zip(this.cloudFoundryClient, this.spaceId)
+                    .flatMap(
+                            function(
+                                    (cloudFoundryClient, spaceId) ->
+                                            getApplicationId(
+                                                    cloudFoundryClient,
+                                                    request.getName(),
+                                                    spaceId)))
+                    .flatMapMany(
+                            applicationId -> getLogsLogCache(this.logCacheClient, applicationId))
+                    .transform(OperationsLogging.log("Get Application Logs"))
+                    .checkpoint();
+        } else {
+            return logs(LogsRequest.builder().name(request.getName()).recent(false).build())
+                    .map(DefaultApplications::toApplicationLog);
+        }
     }
 
     @Override
@@ -673,7 +687,6 @@ public Mono<Void> pushManifestV3(PushManifestV3Request request) {
         } catch (IOException e) {
             throw new RuntimeException("Could not serialize manifest", e);
         }
-
         return Mono.zip(this.cloudFoundryClient, this.spaceId)
                 .flatMap(
                         function(
@@ -1617,6 +1630,32 @@ private static Flux<LogMessage> getLogs(
         }
     }
 
+    private static Flux<ApplicationLog> getLogsLogCache(
+            Mono<LogCacheClient> logCacheClient, String applicationId) {
+        return requestLogsRecentLogCache(logCacheClient, applicationId)
+                .filter(e -> e.getLog() != null)
+                .sort(LOG_MESSAGE_COMPARATOR_LOG_CACHE)
+                .map(
+                        envelope ->
+                                ApplicationLog.builder()
+                                        .sourceId(
+                                                Optional.ofNullable(envelope.getSourceId())
+                                                        .orElse(""))
+                                        .sourceType(
+                                                envelope.getTags().getOrDefault("source_type", ""))
+                                        .instanceId(
+                                                Optional.ofNullable(envelope.getInstanceId())
+                                                        .orElse(""))
+                                        .message(envelope.getLog().getPayloadAsText())
+                                        .timestamp(
+                                                Optional.ofNullable(envelope.getTimestamp())
+                                                        .orElse(0L))
+                                        .logType(
+                                                ApplicationLogType.from(
+                                                        envelope.getLog().getType().name()))
+                                        .build());
+    }
+
     @SuppressWarnings("unchecked")
     private static Map<String, Object> getMetadataRequest(EventEntity entity) {
         Map<String, Optional<Object>> metadata =
@@ -2501,6 +2540,7 @@ private static Flux<TaskResource> requestListTasks(
                                                 .build()));
     }
 
+    @Deprecated
     private static Flux<Envelope> requestLogsRecent(
             Mono<DopplerClient> dopplerClient, String applicationId) {
         return dopplerClient.flatMapMany(
@@ -2509,6 +2549,16 @@ private static Flux<Envelope> requestLogsRecent(
                                 RecentLogsRequest.builder().applicationId(applicationId).build()));
     }
 
+    private static Flux<org.cloudfoundry.logcache.v1.Envelope> requestLogsRecentLogCache(
+            Mono<LogCacheClient> logCacheClient, String applicationId) {
+        return logCacheClient
+                .flatMap(
+                        client ->
+                                client.read(ReadRequest.builder().sourceId(applicationId).build()))
+                .flatMap(response -> Mono.justOrEmpty(response.getEnvelopes()))
+                .flatMapIterable(EnvelopeBatch::getBatch);
+    }
+
     private static Flux<Envelope> requestLogsStream(
             Mono<DopplerClient> dopplerClient, String applicationId) {
         return dopplerClient.flatMapMany(
@@ -2914,6 +2964,17 @@ private static Mono<AbstractApplicationResource> stopApplicationIfNotStopped(
                 : Mono.just(resource);
     }
 
+    private static ApplicationLog toApplicationLog(LogMessage logMessage) {
+        return ApplicationLog.builder()
+                .sourceId(logMessage.getApplicationId())
+                .sourceType(logMessage.getSourceType())
+                .instanceId(logMessage.getSourceInstance())
+                .message(logMessage.getMessage())
+                .timestamp(logMessage.getTimestamp())
+                .logType(ApplicationLogType.from(logMessage.getMessageType().name()))
+                .build();
+    }
+
     private static ApplicationDetail toApplicationDetail(
             List<String> buildpacks,
             SummaryApplicationResponse summaryApplicationResponse,

cloudfoundry-operations/src/test/java/org/cloudfoundry/operations/AbstractOperationsTest.java

@@ -51,6 +51,7 @@
 import org.cloudfoundry.client.v3.spaces.SpacesV3;
 import org.cloudfoundry.client.v3.tasks.Tasks;
 import org.cloudfoundry.doppler.DopplerClient;
+import org.cloudfoundry.logcache.v1.LogCacheClient;
 import org.cloudfoundry.routing.RoutingClient;
 import org.cloudfoundry.routing.v1.routergroups.RouterGroups;
 import org.cloudfoundry.uaa.UaaClient;
@@ -101,6 +102,8 @@ public abstract class AbstractOperationsTest {
 
     protected final DopplerClient dopplerClient = mock(DopplerClient.class, RETURNS_SMART_NULLS);
 
+    protected final LogCacheClient logCacheClient = mock(LogCacheClient.class, RETURNS_SMART_NULLS);
+
     protected final Events events = mock(Events.class, RETURNS_SMART_NULLS);
 
     protected final FeatureFlags featureFlags = mock(FeatureFlags.class, RETURNS_SMART_NULLS);