Skip to content

[Backport 1.28.x] Revised AzureBlobStore deletions. #1457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aaime merged 4 commits into from
Dec 2, 2025
Merged

[Backport 1.28.x] Revised AzureBlobStore deletions. #1457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aab9d53
Revised AzureBlobStore deletions.
axl8713 Nov 20, 2025
e848489
Fixed bulk delete counter.
axl8713 Nov 25, 2025
5a1dbc1
Refactored tile range deletion to be streamed.
axl8713 Nov 27, 2025
83e446d
Refactored tile range deletion to be fully streamed.
axl8713 Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions geowebcache/azureblob/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@
<artifactId>azure-storage-blob</artifactId>
<version>${azure.version}</version>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob-batch</artifactId>
<version>${azure.version}</version>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
Expand Down
124 changes: 84 additions & 40 deletions geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureBlobStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,30 @@

import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Objects.isNull;
import static org.geowebcache.azure.DeleteManager.PAGE_SIZE;

import com.azure.core.util.BinaryData;
import com.azure.storage.blob.models.BlobDownloadContentResponse;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.BlobProperties;
import com.azure.storage.blob.models.BlobStorageException;
import com.azure.storage.blob.specialized.BlockBlobClient;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.geotools.util.logging.Logging;
Expand All @@ -53,13 +56,14 @@
import org.geowebcache.storage.StorageException;
import org.geowebcache.storage.TileObject;
import org.geowebcache.storage.TileRange;
import org.geowebcache.storage.TileRangeIterator;
import org.geowebcache.util.TMSKeyBuilder;
import org.springframework.http.HttpStatus;

public class AzureBlobStore implements BlobStore {

static Logger log = Logging.getLogger(AzureBlobStore.class.getName());
private static final Logger LOG = Logging.getLogger(AzureBlobStore.class.getName());

private static final Pattern TILE_BLOB_NAME_REGEXP = Pattern.compile("(?<z>\\d+)/(?<x>\\d+)/(?<y>\\d+)\\.\\w+$");

private final TMSKeyBuilder keyBuilder;
private final BlobStoreListenerList listeners = new BlobStoreListenerList();
Expand Down Expand Up @@ -190,58 +194,98 @@ public boolean delete(TileRange tileRange) throws StorageException {
return false;
}

// open an iterator oer tile locations, to avoid memory accumulation
final Iterator<long[]> tileLocations = new AbstractIterator<>() {

// TileRange iterator with 1x1 meta tiling factor
private TileRangeIterator trIter = new TileRangeIterator(tileRange, new int[] {1, 1});

@Override
protected long[] computeNext() {
long[] gridLoc = trIter.nextMetaGridLocation(new long[3]);
return gridLoc == null ? endOfData() : gridLoc;
}
};
Stream<BlobItem> blobsToDelete = findTileBlobsToDelete(tileRange, coordsPrefix);

// if no listeners, we don't need to gather extra tile info, use a dedicated fast path
if (listeners.isEmpty()) {
// if there are no listeners, don't bother requesting every tile
// metadata to notify the listeners
Iterator<String> keysIterator = Iterators.transform(
tileLocations, tl -> keyBuilder.forLocation(coordsPrefix, tl, tileRange.getMimeType()));
// split the iteration in parts to avoid memory accumulation
Iterator<List<String>> partition = Iterators.partition(keysIterator, DeleteManager.PAGE_SIZE);

while (partition.hasNext() && !shutDown) {
List<String> locations = partition.next();
deleteManager.deleteParallel(locations);
if (!shutDown) {
deleteManager.deleteStreamed(blobsToDelete);
}

} else {
// if we need to gather info, we'll end up just calling "delete" on each tile
// this is run here instead of inside the delete manager as we need high level info
// about tiles, e.g., TileObject, to inform the listeners
String layerName = tileRange.getLayerName();
String gridSetId = tileRange.getGridSetId();
String format = tileRange.getMimeType().getFormat();
Map<String, String> parameters = tileRange.getParameters();

Iterator<Callable<?>> tilesIterator = Iterators.transform(tileLocations, xyz -> {
TileObject tile = TileObject.createQueryTileObject(layerName, xyz, gridSetId, format, parameters);
Stream<Callable<?>> tilesDeletions = blobsToDelete.map(blobItem -> {
TileObject tile = createTileObject(blobItem, tileRange);
tile.setParametersId(tileRange.getParametersId());
return (Callable<Object>) () -> delete(tile);
return () -> delete(tile);
});
Iterator<List<Callable<?>>> partition = Iterators.partition(tilesIterator, DeleteManager.PAGE_SIZE);

// once a page of callables is ready, run them in parallel on the delete manager
while (partition.hasNext() && !shutDown) {
deleteManager.executeParallel(partition.next());
}
executeParallelDeletions(tilesDeletions);
}

return true;
}

private Stream<BlobItem> findTileBlobsToDelete(TileRange tileRange, String coordsPrefix) {
return IntStream.rangeClosed(tileRange.getZoomStart(), tileRange.getZoomStop())
.boxed()
.flatMap(zoom -> {
String zoomPrefix = coordsPrefix + "/" + zoom;

if (!client.prefixExists(zoomPrefix)) {
// empty level, skipping
return Stream.empty();
}

long[] rangeBoundsAtZoom = tileRange.rangeBounds(zoom);

return client.listBlobs(zoomPrefix)
.filter(tb ->
TILE_BLOB_NAME_REGEXP.matcher(tb.getName()).find())
.filter(tb -> isTileBlobInBounds(tb, rangeBoundsAtZoom));
});
}

private boolean isTileBlobInBounds(BlobItem tileBlob, long[] bounds) {
long minX = bounds[0];
long minY = bounds[1];
long maxX = bounds[2];
long maxY = bounds[3];

long[] index = extractTileIndex(tileBlob);
long tileX = index[0];
long tileY = index[1];

return tileX >= minX && tileX <= maxX && tileY >= minY && tileY <= maxY;
}

private TileObject createTileObject(BlobItem blobItem, TileRange tileRange) {
String layerName = tileRange.getLayerName();
String gridSetId = tileRange.getGridSetId();
String format = tileRange.getMimeType().getFormat();
Map<String, String> parameters = tileRange.getParameters();
return TileObject.createQueryTileObject(layerName, extractTileIndex(blobItem), gridSetId, format, parameters);
}

private long[] extractTileIndex(BlobItem blobItem) {
Matcher matcher = TILE_BLOB_NAME_REGEXP.matcher(blobItem.getName());

if (!matcher.find()) {
throw new IllegalArgumentException("Invalid tile blob name");
}

return new long[] {
Long.parseLong(matcher.group("x")), Long.parseLong(matcher.group("y")), Long.parseLong(matcher.group("z"))
};
}

private void executeParallelDeletions(Stream<Callable<?>> tilesDeletions) throws StorageException {
Iterator<Callable<?>> tilesDeletionsIterator = tilesDeletions.iterator();

while (tilesDeletionsIterator.hasNext() && !shutDown) {

// once a page of callables is ready, run them in parallel on the delete manager
List<Callable<?>> callables = new ArrayList<>(PAGE_SIZE);
for (int i = 0; i < PAGE_SIZE && tilesDeletionsIterator.hasNext(); i++) {
callables.add(tilesDeletionsIterator.next());
}

deleteManager.executeParallel(callables);
}
}

@Override
public boolean get(TileObject obj) throws StorageException {
final String key = keyBuilder.forTile(obj);
Expand Down Expand Up @@ -373,7 +417,7 @@ public boolean rename(String oldLayerName, String newLayerName) throws StorageEx
// revisit: this seems to hold true only for GeoServerTileLayer, "standalone" TileLayers
// return getName() from getId(), as in AbstractTileLayer. Unfortunately the only option
// for non-GeoServerTileLayers would be copy and delete. Expensive.
log.fine("No need to rename layers, AzureBlobStore uses layer id as key root");
LOG.fine("No need to rename layers, AzureBlobStore uses layer id as key root");
if (client.prefixExists(oldLayerName)) {
listeners.sendLayerRenamed(oldLayerName, newLayerName);
}
Expand Down
8 changes: 8 additions & 0 deletions geowebcache/azureblob/src/main/java/org/geowebcache/azure/AzureClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.batch.BlobBatchClient;
import com.azure.storage.blob.batch.BlobBatchClientBuilder;
import com.azure.storage.blob.models.BlobDownloadContentResponse;
import com.azure.storage.blob.models.BlobHttpHeaders;
import com.azure.storage.blob.models.BlobItem;
Expand Down Expand Up @@ -55,6 +57,7 @@ public class AzureClient {

private AzureBlobStoreData configuration;
private final BlobContainerClient container;
private final BlobBatchClient batch;

public AzureClient(AzureBlobStoreData configuration) throws StorageException {
this.configuration = configuration;
Expand All @@ -64,6 +67,7 @@ public AzureClient(AzureBlobStoreData configuration) throws StorageException {

String containerName = configuration.getContainer();
this.container = getOrCreateContainer(serviceClient, containerName);
this.batch = new BlobBatchClientBuilder(serviceClient).buildClient();
} catch (StorageException e) {
throw e;
} catch (RuntimeException e) {
Expand Down Expand Up @@ -284,6 +288,10 @@ public BlobContainerClient getContainer() {
return container;
}

public BlobBatchClient getBatch() {
return batch;
}

public boolean deleteBlob(String key) {
BlockBlobClient metadata = getBlockBlobClient(key);
return metadata.deleteIfExists();
Expand Down
Loading