Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.management.ManagementFactory;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
Expand Down Expand Up @@ -200,6 +201,8 @@ private ContextDefinition computeDefinitionAndClassLoader(
monitorContext(contextLocation, computedDefinition.getMonitorIntervalSeconds());
} catch (IOException e) {
throw new UncheckedIOException(e);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
} else {
computedDefinition = previousDefinition;
Expand All @@ -216,10 +219,11 @@ private ContextDefinition computeDefinitionAndClassLoader(
return computedDefinition;
}

private static ContextDefinition getDefinition(String contextLocation) throws IOException {
private static ContextDefinition getDefinition(String contextLocation)
throws IOException, URISyntaxException {
LOG.trace("Retrieving context definition file from {}", contextLocation);
URL url = new URL(contextLocation);
return ContextDefinition.fromRemoteURL(url);
URI uri = new URI(contextLocation);
return ContextDefinition.fromRemoteURL(uri);
}

private static String newCacheKey(String contextLocation, String contextChecksum) {
Expand Down Expand Up @@ -268,7 +272,7 @@ private void checkMonitoredLocation(String contextLocation, long interval) {
} else {
LOG.trace("Context definition for {} has not changed", contextLocation);
}
} catch (IOException | RuntimeException e) {
} catch (IOException | RuntimeException | URISyntaxException e) {
LOG.error("Error parsing updated context definition at {}. Classloader NOT updated!",
contextLocation, e);
final Stopwatch failureTimer = classloaderFailures.get(contextLocation);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
Expand All @@ -39,8 +39,6 @@
import org.apache.accumulo.classloader.lcc.resolvers.FileResolver;
import org.apache.accumulo.core.cli.Help;
import org.apache.accumulo.start.spi.KeywordExecutable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;

import com.beust.jcommander.Parameter;
import com.google.auto.service.AutoService;
Expand All @@ -66,10 +64,10 @@ static class Opts extends Help {
private static final Gson GSON =
new GsonBuilder().disableJdkUnsafe().setPrettyPrinting().create();

public static ContextDefinition create(int monitorIntervalSecs, URL... sources)
public static ContextDefinition create(int monitorIntervalSecs, URI... sources)
throws IOException {
LinkedHashSet<Resource> resources = new LinkedHashSet<>();
for (URL u : sources) {
for (URI u : sources) {
FileResolver resolver = FileResolver.resolve(u);
try (InputStream is = resolver.getInputStream()) {
String checksum = DIGESTER.digestAsHex(is);
Expand All @@ -79,12 +77,12 @@ public static ContextDefinition create(int monitorIntervalSecs, URL... sources)
return new ContextDefinition(monitorIntervalSecs, resources);
}

public static ContextDefinition fromRemoteURL(final URL url) throws IOException {
final FileResolver resolver = FileResolver.resolve(url);
public static ContextDefinition fromRemoteURL(final URI uri) throws IOException {
final FileResolver resolver = FileResolver.resolve(uri);
try (InputStream is = resolver.getInputStream()) {
var def = GSON.fromJson(new InputStreamReader(is, UTF_8), ContextDefinition.class);
if (def == null) {
throw new EOFException("InputStream does not contain a valid ContextDefinition at " + url);
throw new EOFException("InputStream does not contain a valid ContextDefinition at " + uri);
}
return def;
}
Expand Down Expand Up @@ -160,14 +158,12 @@ public String description() {

@Override
public void execute(String[] args) throws Exception {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory(new Configuration()));

Opts opts = new Opts();
opts.parseArgs(ContextDefinition.class.getName(), args);
URL[] urls = new URL[opts.files.size()];
URI[] urls = new URI[opts.files.size()];
int count = 0;
for (String f : opts.files) {
urls[count++] = new URL(f);
urls[count++] = new URI(f);
}
ContextDefinition def = create(opts.monitorInterval, urls);
System.out.print(def.toJson());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,22 @@
*/
package org.apache.accumulo.classloader.lcc.definition;

import java.net.URL;
import java.net.URI;
import java.util.Objects;

public class Resource {

private URL location;
private URI location;
private String checksum;

public Resource() {}

public Resource(URL location, String checksum) {
public Resource(URI location, String checksum) {
this.location = location;
this.checksum = checksum;
}

public URL getLocation() {
public URI getLocation() {
return location;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,34 +23,37 @@

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URI;
import java.util.Objects;

import com.google.common.base.Preconditions;

public abstract class FileResolver {

public static FileResolver resolve(URL url) throws IOException {
requireNonNull(url, "URL must be supplied");
switch (url.getProtocol()) {
public static FileResolver resolve(URI uri) throws IOException {
requireNonNull(uri, "URI must be supplied");
Preconditions.checkArgument(uri.isAbsolute(), "URI is not absolute : %s", uri);
switch (uri.getScheme()) {
case "http":
case "https":
return new HttpFileResolver(url);
return new HttpFileResolver(uri);
case "file":
return new LocalFileResolver(url);
return new LocalFileResolver(uri);
case "hdfs":
return new HdfsFileResolver(url);
return new HdfsFileResolver(uri);
default:
throw new IOException("Unhandled protocol: " + url.getProtocol());
throw new IOException("Unhandled protocol: " + uri.getScheme());
}
}

private final URL url;
private final URI uri;

protected FileResolver(URL url) {
this.url = url;
protected FileResolver(URI uri) {
this.uri = uri;
}

protected URL getURL() {
return this.url;
protected URI getURI() {
return this.uri;
}

public abstract String getFileName();
Expand All @@ -59,7 +62,7 @@ protected URL getURL() {

@Override
public int hashCode() {
return hash(url);
return hash(uri);
}

@Override
Expand All @@ -74,7 +77,7 @@ public boolean equals(Object obj) {
return false;
}
FileResolver other = (FileResolver) obj;
return Objects.equals(url, other.url);
return Objects.equals(uri, other.uri);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -21,30 +21,28 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import com.google.common.base.Preconditions;

public final class HdfsFileResolver extends FileResolver {

private final Configuration hadoopConf = new Configuration();
private final FileSystem fs;
private final Path path;

protected HdfsFileResolver(URL url) throws IOException {
super(url);
try {
final URI uri = url.toURI();
this.fs = FileSystem.get(uri, hadoopConf);
this.path = fs.makeQualified(new Path(uri));
if (!fs.exists(this.path)) {
throw new IOException("File: " + url + " does not exist.");
}
} catch (URISyntaxException e) {
throw new IOException("Error creating URI from url: " + url, e);
protected HdfsFileResolver(URI uri) throws IOException {
super(uri);

Preconditions.checkArgument(uri.getScheme().equals("hdfs"), "Not HDFS URI : " + uri);

this.fs = FileSystem.get(uri, hadoopConf);
this.path = fs.makeQualified(new Path(uri));
if (!fs.exists(this.path)) {
throw new IOException("File: " + uri + " does not exist.");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,33 @@

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URI;

import com.google.common.base.Preconditions;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

public final class HttpFileResolver extends FileResolver {

protected HttpFileResolver(URL url) throws IOException {
super(url);
protected HttpFileResolver(URI uri) throws IOException {
super(uri);
// Converting to a URL will perform more strict checks, also ensure the protocol is correct.
var url = uri.toURL();
Preconditions.checkArgument(
url.getProtocol().equals("http") || url.getProtocol().equals("https"),
"Not an HTTP url " + url);
}

@Override
public String getFileName() {
String path = getURL().getPath();
String path = getURI().getPath();
return path.substring(path.lastIndexOf("/") + 1);
}

@Override
@SuppressFBWarnings(value = "URLCONNECTION_SSRF_FD",
justification = "user-supplied URL is the intended functionality")
public InputStream getInputStream() throws IOException {
return getURL().openStream();
return getURI().toURL().openStream();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,31 +23,32 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;

import com.google.common.base.Preconditions;

public final class LocalFileResolver extends FileResolver {

private final File file;

public LocalFileResolver(URL url) throws IOException {
super(url);
if (url.getHost() != null && !url.getHost().isBlank()) {
public LocalFileResolver(URI uri) throws IOException {
super(uri);

// Converting to a URL will perform more strict checks, also ensure the protocol is correct.
var url = uri.toURL();
Preconditions.checkArgument(url.getProtocol().equals("file"), "Not a file url : " + url);

if (uri.getHost() != null && !uri.getHost().isBlank()) {
throw new IOException(
"Unsupported file url, only local files are supported. host = " + url.getHost());
"Unsupported file uri, only local files are supported. host = " + uri.getHost());
}
try {
final URI uri = url.toURI();
final Path path = Path.of(uri);
if (Files.notExists(Path.of(uri))) {
throw new IOException("File: " + url + " does not exist.");
}
file = path.toFile();
} catch (URISyntaxException e) {
throw new IOException("Error creating URI from url: " + url, e);

final Path path = Path.of(uri);
if (Files.notExists(path)) {
throw new IOException("File: " + uri + " does not exist.");
}
file = path.toFile();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.FileAlreadyExistsException;
Expand Down Expand Up @@ -194,15 +195,15 @@ private void storeContextDefinition(final ContextDefinition contextDefinition,
* Failures to download are not re-attempted, but will propagate up to the caller.
*/
private Path storeResource(final Resource resource) throws IOException {
final URL url = resource.getLocation();
final FileResolver source = FileResolver.resolve(url);
final URI uri = resource.getLocation();
final FileResolver source = FileResolver.resolve(uri);
final String baseName = localResourceName(source.getFileName(), resource.getChecksum());
final Path destinationPath = resourcesDir.resolve(baseName);
final Path tempPath = resourcesDir.resolve(tempName(baseName));
final Path downloadingProgressPath = resourcesDir.resolve("." + baseName + ".downloading");

if (Files.exists(destinationPath)) {
LOG.trace("Resource {} is already cached at {}", url, destinationPath);
LOG.trace("Resource {} is already cached at {}", uri, destinationPath);
verifyDownload(resource, destinationPath, null);
try {
// clean up any in progress files that may have been left behind by previous failed attempts
Expand Down Expand Up @@ -235,9 +236,9 @@ private Path storeResource(final Resource resource) throws IOException {
var task = new FutureTask<Void>(() -> downloadFile(source, tempPath, resource), null);
var t = new Thread(task);
t.setDaemon(true);
t.setName("downloading " + url + " to " + tempPath);
t.setName("downloading " + uri + " to " + tempPath);

LOG.trace("Storing remote resource {} locally at {} via temp file {}", url, destinationPath,
LOG.trace("Storing remote resource {} locally at {} via temp file {}", uri, destinationPath,
tempPath);
t.start();
try {
Expand All @@ -258,11 +259,11 @@ private Path storeResource(final Resource resource) throws IOException {
task.cancel(true);
Thread.currentThread().interrupt();
throw new IllegalStateException(
"Thread was interrupted while waiting on resource to copy from " + url + " to "
"Thread was interrupted while waiting on resource to copy from " + uri + " to "
+ tempPath,
e);
} catch (ExecutionException e) {
throw new IllegalStateException("Error copying resource from " + url + " to " + tempPath,
throw new IllegalStateException("Error copying resource from " + uri + " to " + tempPath,
e);
}
}
Expand Down
Loading