Implement filters endpoint

Author: JulianVennen

CHANGELOG.md

@@ -1,3 +1,15 @@
+# v6.2.0
+
+## Improvements
+- Filter usernames and session tokens in uploaded logs as defined by the instance filters
+- Fix some requests hanging when their body is being read 
+- Trim beginning of logs before applying limits to avoid counting leading newlines and whitespace towards the limit
+
+## Deprecations
+- The pattern constants in `Log` are now deprecated and only serve as fallbacks when the filters endpoint is not available on the specified instance.
+- `Log#getContent(Limits)` is now deprecated. The new `getContent(FilterList)` method is internal and might change without notice.
+- `LogReader#readContents(Limits)` is now deprecated. The new `readContents(FilterList)` method is internal and might change without notice.
+
 # v6.1.0
 
 ## Improvements

src/jmh/java/gs/mlco/api/LimitedReaderBenchmark.java

@@ -51,7 +51,7 @@ public int testWithBufferedInputStreamLineLimit() throws IOException {
     }
 
     private int test(Reader in, Integer byteLimit, Integer lineLimit) throws IOException {
-        try (Reader reader = new LimitedReader(in, new Limits(1, byteLimit, lineLimit))) {
+        try (Reader reader = new LimitedReader(in, byteLimit, lineLimit, true)) {
             StringWriter writer = new StringWriter();
             reader.transferTo(writer);
             return writer.toString().length();

src/main/java/gs/mclo/api/Instance.java

@@ -93,6 +93,14 @@ public String getLogUrl(String id) {
         return apiBaseUrl + "1/log/" + id;
     }
 
+    /**
+     * Get the URL for fetching available filters
+     * @return the URL for fetching available filters (e.g. <a href="https://api.mclo.gs/1/filters">https://api.mclo.gs/1/filters</a>)
+     */
+    public String getFiltersUrl() {
+        return apiBaseUrl + "1/filters";
+    }
+
     private String ensureEndsWithSlash(String url) {
         if (!url.endsWith("/"))
             url += "/";

src/main/java/gs/mclo/api/Log.java

@@ -1,28 +1,32 @@
 package gs.mclo.api;
 
 import gs.mclo.api.data.Metadata;
+import gs.mclo.api.internal.filter.*;
 import gs.mclo.api.reader.FileLogReader;
 import gs.mclo.api.reader.LogReader;
 import gs.mclo.api.reader.StringLogReader;
 import gs.mclo.api.response.Limits;
+import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 public class Log {
     /**
      * pattern for IPv4 addresses
      */
+    @Deprecated
     public static final Pattern IPV4_PATTERN = Pattern.compile("(?<!([0-9]|-|\\w))(?:[1-2]?[0-9]{1,2}\\.){3}[1-2]?[0-9]{1,2}(?!([0-9]|-|\\w))");
 
     /**
      * whitelist patterns for IPv4 addresses
      */
+    @Deprecated
     public static final Pattern[] IPV4_FILTER = new Pattern[]{
             Pattern.compile("127\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}"),
             Pattern.compile("0\\.0\\.0\\.0"),
@@ -33,11 +37,13 @@ public class Log {
     /**
      * pattern for IPv6 addresses
      */
+    @Deprecated
     public static final Pattern IPV6_PATTERN = Pattern.compile("(?<!([0-9]|-|\\w))(?:[0-9a-f]{0,4}:){7}[0-9a-f]{0,4}(?!([0-9]|-|\\w))", Pattern.CASE_INSENSITIVE);
 
     /**
      * whitelist patterns for IPv4 addresses
      */
+    @Deprecated
     public static final Pattern[] IPV6_FILTER = new Pattern[]{
             Pattern.compile("[0:]+1?"),
     };
@@ -87,84 +93,33 @@ public Log(LogReader reader) {
     }
 
     /**
-     * remove IP addresses
-     */
-    private String filter(String content) {
-        content = this.filterIPv4(content);
-        return this.filterIPv6(content);
-    }
-
-    /**
-     * remove IPv4 addresses
-     */
-    private String filterIPv4(String content) {
-        Matcher matcher = IPV4_PATTERN.matcher(content);
-        StringBuilder sb = new StringBuilder();
-        while (matcher.find()) {
-            if (isWhitelistedIPv4(matcher.group())) {
-                continue;
-            }
-            matcher.appendReplacement(sb, "**.**.**.**");
-        }
-        matcher.appendTail(sb);
-        return sb.toString();
-    }
-
-    /**
-     * does this IPv4 address match any whitelist filters
-     *
-     * @param s string to test
-     * @return matches
-     */
-    private boolean isWhitelistedIPv4(String s) {
-        for (Pattern filter : IPV4_FILTER) {
-            if (s.matches(filter.pattern())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * remove IPv6 addresses
+     * @return log content
      */
-    private String filterIPv6(String content) {
-        Matcher matcher = IPV6_PATTERN.matcher(content);
-        StringBuilder sb = new StringBuilder();
-        while (matcher.find()) {
-            if (isWhitelistedIPv6(matcher.group())) {
-                continue;
-            }
-            matcher.appendReplacement(sb, "****:****:****:****:****:****:****:****");
-        }
-        matcher.appendTail(sb);
-        return sb.toString();
-    }
-
-    /**
-     * does this IPv6 address match any whitelist filters
-     *
-     * @param s string to test
-     * @return matches
-     */
-    private boolean isWhitelistedIPv6(String s) {
-        for (Pattern filter : IPV6_FILTER) {
-            if (s.matches(filter.pattern())) {
-                return true;
-            }
-        }
-        return false;
+    @Deprecated
+    public String getContent(Limits limits) throws IOException {
+        return getContent(new FilterList(new Filter[]{
+                new TrimFilter(),
+                new LimitBytesFilter(limits.getMaxLength()),
+                new LimitLinesFilter(limits.getMaxLines()),
+                legacyRegex(IPV4_PATTERN, IPV4_FILTER, "**.**.**.**"),
+                legacyRegex(IPV6_PATTERN, IPV6_FILTER, "****:****:****:****:****:****:****:****"),
+        }));
     }
 
     /**
      * @return log content
      */
-    public String getContent(Limits limits) throws IOException {
+    @ApiStatus.Internal
+    public String getContent(FilterList filters) throws IOException {
         if (content != null) {
             return content;
         }
 
-        this.content = this.filter(reader.readContents(limits));
+        content = reader.readContents(filters);
+        for (var filter : filters.getFilters()) {
+            content = filter.apply(content);
+        }
+
         return content;
     }
 
@@ -218,4 +173,10 @@ public Log addMetadata(Metadata<?> data) {
         this.metadata.add(data);
         return this;
     }
+
+    private RegexFilter legacyRegex(Pattern pattern, Pattern[] exemptions, String replacement) {
+        return new RegexFilter(new ReplacingRegexPattern[]{
+                new ReplacingRegexPattern(pattern.pattern(), new char[]{}, replacement),
+        }, Arrays.stream(exemptions).map(p -> new RegexPattern(p.pattern(), new char[]{})).toArray(RegexPattern[]::new));
+    }
 }

src/main/java/gs/mclo/api/MclogsClient.java

@@ -5,12 +5,11 @@
 import gs.mclo.api.data.LogField;
 import gs.mclo.api.internal.JsonBodyHandler;
 import gs.mclo.api.internal.RequestBuilder;
+import gs.mclo.api.internal.filter.*;
+import gs.mclo.api.internal.gson.FilterTypeAdapterFactory;
 import gs.mclo.api.internal.gson.InstantTypeAdapter;
 import gs.mclo.api.reader.FileLogReader;
-import gs.mclo.api.response.GetLogResponse;
-import gs.mclo.api.response.InsightsResponse;
-import gs.mclo.api.response.Limits;
-import gs.mclo.api.response.UploadLogResponse;
+import gs.mclo.api.response.*;
 import org.jetbrains.annotations.ApiStatus;
 import org.jetbrains.annotations.Nullable;
 
@@ -27,6 +26,7 @@
 public class MclogsClient {
     private final Gson gson = new GsonBuilder()
             .registerTypeAdapter(Instant.class, new InstantTypeAdapter())
+            .registerTypeAdapterFactory(new FilterTypeAdapterFactory())
             .create();
     private final HttpClient httpClient = HttpClient.newBuilder()
             .version(HttpClient.Version.HTTP_2)
@@ -35,6 +35,7 @@ public class MclogsClient {
 
     private Instance instance = new Instance();
     private @Nullable Limits limits;
+    private @Nullable FilterList filters;
     private final RequestBuilder requestBuilder = new RequestBuilder(gson);
 
     /**
@@ -143,13 +144,13 @@ public MclogsClient setInstance(Instance instance) {
      * @return the response
      */
     public CompletableFuture<UploadLogResponse> uploadLog(Log log) {
-        return getLimitsOrDefault().thenCompose(limits -> {
+        return getFiltersOrDefault().thenCompose(filters -> {
             if (log.getSource() == null) {
                 log.setSource(requestBuilder.getProjectName());
             }
 
             try {
-                HttpRequest request = requestBuilder.uploadRequest(instance.getLogUploadUrl(), log, limits);
+                HttpRequest request = requestBuilder.uploadRequest(instance.getLogUploadUrl(), log, filters);
                 return asyncRequest(request, UploadLogResponse.class);
             } catch (IOException e) {
                 return CompletableFuture.failedFuture(e);
@@ -211,9 +212,9 @@ public CompletableFuture<InsightsResponse> getInsights(String logId) {
      * @return the insights of the log
      */
     public CompletableFuture<InsightsResponse> analyseLog(Log log) {
-        return this.getLimitsOrDefault().thenCompose(limits -> {
+        return this.getFiltersOrDefault().thenCompose(filters -> {
             try {
-                HttpRequest request = requestBuilder.uploadRequest(instance.getLogAnalysisUrl(), log, limits);
+                HttpRequest request = requestBuilder.uploadRequest(instance.getLogAnalysisUrl(), log, filters);
                 return asyncRequest(request, InsightsResponse.class);
             } catch (IOException e) {
                 return CompletableFuture.failedFuture(e);
@@ -298,6 +299,29 @@ public CompletableFuture<Void> deleteLog(String id, String token) {
         });
     }
 
+    /**
+     * Get the filters that should be applied when uploading logs
+     *
+     * @return an array of filters
+     */
+    CompletableFuture<FilterList> getFilters() {
+        var filters = this.filters;
+        if (filters != null) {
+            return CompletableFuture.completedFuture(filters);
+        }
+
+        HttpRequest request = requestBuilder.request(instance.getFiltersUrl())
+                .header("Accept", "application/json")
+                .GET()
+                .build();
+        return asyncRequest(request, Filter[].class)
+                .thenApply(FilterList::new)
+                .thenApply(result -> {
+                    this.filters = result;
+                    return result;
+                });
+    }
+
     /**
      * List logs in the {@code logs} subdirectory of a path
      *
@@ -330,6 +354,7 @@ public String[] listCrashReportsInDirectory(Path directory) {
 
     /**
      * List all files in a directory that match the allowed file name pattern
+     *
      * @param directory directory to list files from
      * @return array of file names that can be uploaded
      */
@@ -346,6 +371,7 @@ public String[] listFilesInDirectory(File directory) {
 
     /**
      * List all files in a directory that match the allowed file name pattern
+     *
      * @param directory directory to list files from
      * @return array of file names that can be uploaded
      */
@@ -368,8 +394,12 @@ private <T> CompletableFuture<T> asyncRequest(HttpRequest request, Class<T> resp
                 .thenApply(HttpResponse::body);
     }
 
-    private CompletableFuture<Limits> getLimitsOrDefault() {
-        return this.getLimits().exceptionally(t -> Limits.DEFAULT);
+    private CompletableFuture<FilterList> getFiltersOrDefault() {
+        return this.getFilters().exceptionally(t -> new FilterList(new Filter[]{
+                new TrimFilter(),
+                new LimitBytesFilter(Limits.DEFAULT.getMaxLength()),
+                new LimitLinesFilter(Limits.DEFAULT.getMaxLines())
+        }));
     }
 
     @ApiStatus.Internal

src/main/java/gs/mclo/api/internal/JsonBodyHandler.java

@@ -1,11 +1,13 @@
 package gs.mclo.api.internal;
 
 import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
 import gs.mclo.api.APIException;
 import gs.mclo.api.MclogsClient;
 import org.jetbrains.annotations.Nullable;
 
 import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CompletionException;
 
 public final class JsonBodyHandler<T> implements HttpResponse.BodyHandler<T> {
@@ -19,15 +21,17 @@ public JsonBodyHandler(MclogsClient client, Class<T> clazz) {
 
     @Override
     public HttpResponse.BodySubscriber<@Nullable T> apply(HttpResponse.ResponseInfo responseInfo) {
-        return HttpResponse.BodySubscribers.mapping(new JsonElementBodySubscriber(client.gson()), x -> map(x, responseInfo));
+        return HttpResponse.BodySubscribers.mapping(HttpResponse.BodySubscribers.ofString(StandardCharsets.UTF_8), x -> map(x, responseInfo));
     }
 
-    @Nullable T map(JsonElement element, HttpResponse.ResponseInfo responseInfo) {
-        if (!element.isJsonObject()) {
+    @Nullable T map(String json, HttpResponse.ResponseInfo responseInfo) {
+        var element = JsonParser.parseString(json);
+        if (element.isJsonObject()) {
+            checkError(element, responseInfo);
+        } else if (!element.isJsonArray()) {
             throw new CompletionException(new APIException("Invalid API response (Status code: " + responseInfo.statusCode() + ")", responseInfo.statusCode()));
         }
 
-        checkError(element, responseInfo);
 
         if (clazz == Void.class) {
             return null;