Add tools for ContextDefinition create and cleanup

.github/workflows/maven-on-demand.yaml

@@ -50,7 +50,7 @@ jobs:
       timeout-minutes: 345
       run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ github.event.inputs.goals }}
       env:
-        MAVEN_OPTS: -Djansi.force=true
+        MAVEN_OPTS: -Djansi.force=true -Dapache.snapshots=true
     - name: Upload unit test results
       if: ${{ failure() }}
       uses: actions/upload-artifact@v4

.github/workflows/maven.yaml

@@ -45,7 +45,7 @@ jobs:
     - name: Build with Maven (verify javadoc:jar)
       run: mvn -B -V -e -ntp "-Dstyle.color=always" verify javadoc:jar -DskipFormat -DverifyFormat
       env:
-        MAVEN_OPTS: -Djansi.force=true
+        MAVEN_OPTS: -Djansi.force=true -Dapache.snapshots=true
     - name: Upload unit test results
       if: ${{ failure() }}
       uses: actions/upload-artifact@v4

modules/local-caching-classloader/README.md

@@ -122,9 +122,18 @@ download, and any modification will likely cause unexpected behavior.
 ## Creating a ContextDefinition file
 
 Users may take advantage of the `ContextDefinition.create(int,URL[])` method to
-construct a `ContextDefinition` object. This will calculate the checksums of
-the classpath elements. `ContextDefinition.toJson()` can be used to serialize
-the `ContextDefinition` to a `String` to store in a file.
+construct a `ContextDefinition` object, programmatically. This will calculate
+the checksums of the classpath elements. `ContextDefinition.toJson()` can be
+used to serialize the `ContextDefinition` to a `String` to store in a file.
+
+Alternatively, if this library's jar is built and placed onto Accumulo's
+`CLASSPATH`, then one can run `bin/accumulo create-context-definition` to
+create the ContextDefinition json file using the command-line. The resulting
+json is printed to stdout and can be redirected to a file. The command takes
+two arguments:
+
+1. the monitor interval, in seconds (e.g. `-i 300`), and
+2. a list of file URLs (e.g. `hdfs://host:port/path/to/one.jar file://host/path/to/two.jar`)
 
 ## Updating a ContextDefinition file
 
@@ -177,12 +186,22 @@ unused files from the cache. While the context definition JSON files are always
 safe to delete, it is not recommended to do so for any that are still in use,
 because they can be useful for troubleshooting.
 
+To aid in this task, a JMX MXBean has been created to expose the files that are
+still referenced by the classloaders that have been created by this factory and
+currently still exist in the system. For an example of how to use this MXBean,
+please see the test method
+`MiniAccumuloClusterClassLoaderFactoryTest.getReferencedFiles`. This method
+attaches to the local Accumulo JVM processes to get the set of referenced
+files. It should be safe to delete files that are located in the local cache
+directory (set by property `general.custom.classloader.lcc.cache.dir`) that are
+NOT in the set of referenced files, so long as no new classloaders have been
+created that reference the files being deleted.
+
 **IMPORTANT**: as mentioned earlier, it is not safe to delete resource files
 that are still referenced by any `ClassLoader` instances. Each `ClassLoader`
 instance assumes that the locally cached resources exist and can be read. They
-will not attempt to download any files.  Downloading and verifying files only
-occurs when `ClassLoader` instances are initially created for a context
-definition.
+will not attempt to download any files.  Downloading files only occurs when
+`ClassLoader` instances are initially created for a context definition.
 
 ## Accumulo Configuration
 

modules/local-caching-classloader/pom.xml

@@ -51,6 +51,17 @@
       <artifactId>spotbugs-annotations</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <!-- needed for annotation processor during compile, but not after -->
+      <groupId>com.google.auto.service</groupId>
+      <artifactId>auto-service</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>com.beust</groupId>
+      <artifactId>jcommander</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>com.github.ben-manes.caffeine</groupId>
       <artifactId>caffeine</artifactId>
@@ -82,6 +93,12 @@
       <artifactId>accumulo-core</artifactId>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <!-- provided by accumulo -->
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-start</artifactId>
+      <scope>provided</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client-api</artifactId>

modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/LocalCachingContextClassLoaderFactory.java

@@ -22,12 +22,15 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.lang.management.ManagementFactory;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executors;
@@ -36,8 +39,16 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
 
+import javax.management.InstanceAlreadyExistsException;
+import javax.management.MBeanRegistrationException;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.NotCompliantMBeanException;
+
 import org.apache.accumulo.classloader.lcc.definition.ContextDefinition;
 import org.apache.accumulo.classloader.lcc.definition.Resource;
+import org.apache.accumulo.classloader.lcc.jmx.ContextClassLoaders;
+import org.apache.accumulo.classloader.lcc.jmx.ContextClassLoadersMXBean;
 import org.apache.accumulo.classloader.lcc.util.DeduplicationCache;
 import org.apache.accumulo.classloader.lcc.util.LccUtils;
 import org.apache.accumulo.classloader.lcc.util.LocalStore;
@@ -97,7 +108,7 @@ public class LocalCachingContextClassLoaderFactory implements ContextClassLoader
 
   // to keep this coherent with the contextDefs, updates to this should be done in the compute
   // method of contextDefs
-  private final DeduplicationCache<String,URL[],URLClassLoader> classloaders =
+  private static final DeduplicationCache<String,URL[],URLClassLoader> classloaders =
       new DeduplicationCache<>(LccUtils::createClassLoader, Duration.ofHours(24), null);
 
   private final AtomicReference<LocalStore> localStore = new AtomicReference<>();
@@ -144,6 +155,16 @@ public void init(ContextClassLoaderEnvironment env) {
       throw new UncheckedIOException("Unable to create the local storage area at " + baseCacheDir,
           e);
     }
+    try {
+      MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+      mbs.registerMBean(new ContextClassLoaders(), ContextClassLoadersMXBean.getObjectName());
+    } catch (MalformedObjectNameException | MBeanRegistrationException
+        | NotCompliantMBeanException e) {
+      throw new IllegalStateException("Error registering MBean", e);
+    } catch (InstanceAlreadyExistsException e) {
+      // instance was re-init'd. This is likely to happen during tests
+      // can ignore as no issue here
+    }
   }
 
   @Override
@@ -279,4 +300,16 @@ private void checkMonitoredLocation(String contextLocation, long interval) {
 
   }
 
+  public static Map<String,List<String>> getReferencedFiles() {
+    final Map<String,List<String>> referencedContexts = new HashMap<>();
+    classloaders.forEach((cacheKey, cl) -> {
+      List<String> files = new ArrayList<>();
+      for (URL u : cl.getURLs()) {
+        files.add(u.toString());
+      }
+      referencedContexts.put(cacheKey, files);
+    });
+    return referencedContexts;
+  }
+
 }

modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/definition/ContextDefinition.java

@@ -28,21 +28,41 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.Supplier;
 
 import org.apache.accumulo.classloader.lcc.resolvers.FileResolver;
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.start.spi.KeywordExecutable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
 
+import com.beust.jcommander.Parameter;
+import com.google.auto.service.AutoService;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Suppliers;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 
-public class ContextDefinition {
+@AutoService(KeywordExecutable.class)
+public class ContextDefinition implements KeywordExecutable {
 
+  static class Opts extends Help {
+    @Parameter(names = {"-i", "--interval"}, required = true,
+        description = "monitor interval (in seconds)", arity = 1, order = 1)
+    int monitorInterval;
+
+    @Parameter(required = true, description = "classpath element URL (<url>[ <url>...])",
+        arity = -1, order = 2)
+    public List<String> files = new ArrayList<>();
+  }
+
+  // pretty-print uses Unix newline
   private static final Gson GSON =
       new GsonBuilder().disableJdkUnsafe().setPrettyPrinting().create();
 
@@ -122,6 +142,34 @@ public String getChecksum() {
   }
 
   public String toJson() {
-    return GSON.toJson(this);
+    // GSON pretty print uses Unix line-endings, and may or may not have a trailing one, so
+    // ensure a trailing one exists, so it's included in checksum computations and in
+    // any files written from this (for better readability)
+    return GSON.toJson(this).stripTrailing() + "\n";
+  }
+
+  @Override
+  public String keyword() {
+    return "create-context-definition";
+  }
+
+  @Override
+  public String description() {
+    return "Creates and prints a Context Definition";
+  }
+
+  @Override
+  public void execute(String[] args) throws Exception {
+    URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(new Configuration()));
+
+    Opts opts = new Opts();
+    opts.parseArgs(ContextDefinition.class.getName(), args);
+    URL[] urls = new URL[opts.files.size()];
+    int count = 0;
+    for (String f : opts.files) {
+      urls[count++] = new URL(f);
+    }
+    ContextDefinition def = create(opts.monitorInterval, urls);
+    System.out.print(def.toJson());
   }
 }

modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/jmx/ContextClassLoaders.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.lcc.jmx;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.classloader.lcc.LocalCachingContextClassLoaderFactory;
+
+public class ContextClassLoaders implements ContextClassLoadersMXBean {
+
+  @Override
+  public Map<String,List<String>> getReferencedFiles() {
+    return LocalCachingContextClassLoaderFactory.getReferencedFiles();
+  }
+
+}

modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/jmx/ContextClassLoadersMXBean.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.classloader.lcc.jmx;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+public interface ContextClassLoadersMXBean {
+
+  static ObjectName getObjectName() throws MalformedObjectNameException {
+    return new ObjectName("org.apache.accumulo.classloader:type=ContextClassLoaders");
+  }
+
+  Map<String,List<String>> getReferencedFiles();
+
+}

modules/local-caching-classloader/src/main/java/org/apache/accumulo/classloader/lcc/util/DeduplicationCache.java

@@ -21,6 +21,7 @@
 import static java.util.Objects.requireNonNull;
 
 import java.time.Duration;
+import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -75,4 +76,13 @@ public boolean anyMatch(final Predicate<KEY> keyPredicate) {
     return canonicalWeakValuesCache.asMap().keySet().stream().anyMatch(keyPredicate);
   }
 
+  public void forEach(final BiConsumer<KEY,VALUE> consumer) {
+    canonicalWeakValuesCache.cleanUp();
+    canonicalWeakValuesCache.asMap().forEach((k, v) -> {
+      if (v != null) {
+        consumer.accept(k, v);
+      }
+    });
+  }
+
 }