Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
6.0-alpha1
* Rework ZSTD dictionary compression logic to create a trainer per training (CASSANDRA-21209)
* Ensure schema created before 2.1 without tableId in folder name can be loaded in SnapshotLoader (CASSANDRA-21173)
* Improve performance when calculating settled placements during range movements (CASSANDRA-21144)
* Make shadow gossip round parameters configurable for testing (CASSANDRA-21149)
* Avoid potential gossip thread deadlock during decommission (CASSANDRA-21143)
Expand All @@ -16,7 +17,7 @@
* Allow overriding compaction strategy parameters during startup (CASSANDRA-21169)
* Introduce created_at column to system_distributed.compression_dictionaries (CASSANDRA-21178)
* Be able to detect and remove orphaned compression dictionaries (CASSANDRA-21157)
* Fix BigTableVerifier to only read a data file during extended verification (CASSANDRA-21150)
* Fix BigTableVerifier to only read a data file during extended verification (CASSANDRA-21150)
* Reduce memory allocation during transformation of BatchStatement to Mutation (CASSANDRA-21141)
* Direct I/O support for compaction reads (CASSANDRA-19987)
* Support custom StartupCheck implementations via SPI (CASSANDRA-21093)
Expand Down
19 changes: 11 additions & 8 deletions src/java/org/apache/cassandra/service/CassandraDaemon.java
Original file line number Diff line number Diff line change
Expand Up @@ -272,14 +272,6 @@ protected void setup()
Keyspace.setInitialized();
CommitLog.instance.start();

SnapshotManager.instance.start(false);
SnapshotManager.instance.clearExpiredSnapshots();
SnapshotManager.instance.clearEphemeralSnapshots();
SnapshotManager.instance.resumeSnapshotCleanup();
SnapshotManager.instance.registerMBean();

// clearing of snapshots above here will in fact clear all ephemeral snapshots
// which were cleared as part of startup checks before CASSANDRA-18111
runStartupChecks();

try
Expand All @@ -306,6 +298,17 @@ protected void setup()
exitOrFail(e.returnCode, e.getMessage(), e.getCause());
}

// SnapshotManager is started after Startup.initialize so that ClusterMetadata is available,
// allowing SnapshotLoader to resolve tableId for pre-2.1 tables via ColumnFamilyStore.
// This means ephemeral/expired snapshot cleanup now runs later in startup than before
// CASSANDRA-21173. No default StartupChecks depend on snapshot state, but custom
// StartupCheck implementations that check disk space may observe stale snapshots.
SnapshotManager.instance.start(false);
SnapshotManager.instance.clearExpiredSnapshots();
SnapshotManager.instance.clearEphemeralSnapshots();
SnapshotManager.instance.resumeSnapshotCleanup();
SnapshotManager.instance.registerMBean();

QueryProcessor.registerStatementInvalidatingListener();

try
Expand Down
46 changes: 44 additions & 2 deletions src/java/org/apache/cassandra/service/snapshot/SnapshotLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,19 @@
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import javax.annotation.Nullable;

import com.google.common.annotations.VisibleForTesting;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.tcm.ClusterMetadata;

import static org.apache.cassandra.db.Directories.SNAPSHOT_SUBDIR;
import static org.apache.cassandra.service.snapshot.TableSnapshot.buildSnapshotId;
Expand All @@ -56,7 +61,9 @@ public class SnapshotLoader
{
private static final Logger logger = LoggerFactory.getLogger(SnapshotLoader.class);

static final Pattern SNAPSHOT_DIR_PATTERN = Pattern.compile("(?<keyspace>\\w+)/(?<tableName>\\w+)-(?<tableId>[0-9a-f]{32})/snapshots/(?<tag>.+)$");
static final Pattern SNAPSHOT_DIR_PATTERN = Pattern.compile("(?<keyspace>\\w+)/(?<tableName>\\w+)" +
"(-(?<tableId>[0-9a-f]{32}))?" +
"/snapshots/(?<tag>.+)$");

private final Collection<Path> dataDirectories;

Expand Down Expand Up @@ -151,12 +158,47 @@ private void loadSnapshotFromDir(Matcher snapshotDirMatcher, Path snapshotDir)
{
String keyspaceName = snapshotDirMatcher.group("keyspace");
String tableName = snapshotDirMatcher.group("tableName");
UUID tableId = parseUUID(snapshotDirMatcher.group("tableId"));
final UUID tableId = maybeDetermineTableId(snapshotDirMatcher, snapshotDir, keyspaceName, tableName);
String tag = snapshotDirMatcher.group("tag");
String snapshotId = buildSnapshotId(keyspaceName, tableName, tableId, tag);
TableSnapshot.Builder builder = snapshots.computeIfAbsent(snapshotId, k -> new TableSnapshot.Builder(keyspaceName, tableName, tableId, tag));
builder.addSnapshotDir(new File(snapshotDir).toAbsolute());
}

private @Nullable UUID maybeDetermineTableId(Matcher snapshotDirMatcher, Path snapshotDir, String keyspaceName, String tableName)
{
final UUID tableId;
if (snapshotDirMatcher.group("tableId") == null)
{
logger.debug("Snapshot directory without tableId found (pre-2.1 format): {}", snapshotDir);
// If we don't have a tableId in folder name (e.g pre 2.1 created table)
// Then attempt to get tableId from CFS on startup
// falling back to null is fine as it still yields a unique result in buildSnapshotId for pre-2.1 table
if (Keyspace.isInitialized() && ClusterMetadata.currentNullable() != null)
{
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspaceName, tableName);
tableId = cfs != null && cfs.metadata != null && cfs.metadata.id != null
? cfs.metadata.id.asUUID()
: null;
if (tableId == null)
{
logger.warn("Snapshot directory without tableId found (pre-2.1 format), " +
"unable to resolve table id from column family, defaulting to null, snapshot dir: {}", snapshotDir);
}
}
else
{
logger.warn("Snapshot directory without tableId found (pre-2.1 format), " +
"TCM or keyspace is not initialized or there is a schema missing, defaulting to null, snapshot dir: {}", snapshotDir);
tableId = null;
}
}
else
{
tableId = parseUUID(snapshotDirMatcher.group("tableId"));
}
return tableId;
}
}

public Set<TableSnapshot> loadSnapshots(String keyspace)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import java.util.UUID;
import java.util.stream.Stream;

import javax.annotation.Nullable;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -68,6 +70,9 @@ public class TableSnapshot
private final String keyspaceName;
private final String tableName;
private final String keyspaceTable;
// tableId may be null under some rare circumstance namely pre-2.1 table
// whose snapshot is loaded upon startup rather than created while the jvm is running
@Nullable
private final UUID tableId;
private final String tag;
private final boolean ephemeral;
Expand Down Expand Up @@ -121,6 +126,7 @@ public void complete()
* Unique identifier of a snapshot. Used
* only to deduplicate snapshots internally,
* not exposed externally.
* table_id may be empty for tables created prior to 2.1
* <p>
* Format: "$ks:$table_name:$table_id:$tag"
*/
Expand Down Expand Up @@ -650,9 +656,9 @@ private void writeManifest(SnapshotManifest snapshotManifest, File manifestFile)
}
}

protected static String buildSnapshotId(String keyspaceName, String tableName, UUID tableId, String tag)
protected static String buildSnapshotId(String keyspaceName, String tableName, @Nullable UUID tableId, String tag)
{
return String.format("%s:%s:%s:%s", keyspaceName, tableName, tableId, tag);
return String.format("%s:%s:%s:%s", keyspaceName, tableName, tableId == null ? "" : tableId, tag);
}

public static Set<Descriptor> getSnapshotDescriptors(String keyspace, String table, String tag)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.utils.Clock;

Expand Down Expand Up @@ -75,6 +76,7 @@ public class SnapshotLoaderTest
public static void setup()
{
DatabaseDescriptor.daemonInitialization();
Keyspace.setInitialized();
}

@Test
Expand All @@ -86,6 +88,9 @@ public void testMatcher()
String TABLE_SNAPSHOT = "/Users/pmottagomes/.ccm/test/node1/data0/ks/my_table-1a025b40c58f11eca526336fc2c671ab/snapshots/test";
assertThat(SNAPSHOT_DIR_PATTERN.matcher(TABLE_SNAPSHOT).find()).isTrue();

String TABLE_SNAPSHOT_PRE_21 = "/Users/pmottagomes/.ccm/test/node1/data0/ks/my_table/snapshots/test";
assertThat(SNAPSHOT_DIR_PATTERN.matcher(TABLE_SNAPSHOT_PRE_21).find()).isTrue();

String DROPPED_SNAPSHOT = "/Users/pmottagomes/.ccm/test/node1/data0/ks/my_table-e5c58330c58d11eca526336fc2c671ab/snapshots/dropped-1650997415751-my_table";
assertThat(SNAPSHOT_DIR_PATTERN.matcher(DROPPED_SNAPSHOT).find()).isTrue();
}
Expand Down Expand Up @@ -124,24 +129,20 @@ public void testSnapshotsWithoutManifests() throws IOException
for (String dataDir : DATA_DIRS)
{
firstSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1));
secondSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2));
secondSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_1, TABLE2_NAME, Directories.SNAPSHOT_SUBDIR, TAG2));
thirdSnapshotDirs.add(createDir(baseDir, dataDir, KEYSPACE_2, tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3));
}

Instant createdAt = Instant.ofEpochMilli(Clock.Global.currentTimeMillis());

createManifests(firstSnapshotDirs, createdAt);
createManifests(secondSnapshotDirs, createdAt);
createManifests(thirdSnapshotDirs, createdAt);

// Verify all 3 snapshots are found correctly from data directories
SnapshotLoader loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
Paths.get(baseDir.toString(), DATA_DIR_2),
Paths.get(baseDir.toString(), DATA_DIR_3)));
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, createdAt, null, firstSnapshotDirs, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, createdAt, null, secondSnapshotDirs, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, null, TAG2, createdAt, null, secondSnapshotDirs, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, createdAt, null, thirdSnapshotDirs, false));

// Verify snapshot loading for a specific keyspace
Expand All @@ -152,7 +153,7 @@ public void testSnapshotsWithoutManifests() throws IOException
snapshots = loader.loadSnapshots(KEYSPACE_1);
assertThat(snapshots).hasSize(2);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, createdAt, null, firstSnapshotDirs, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, createdAt, null, secondSnapshotDirs, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, null, TAG2, createdAt, null, secondSnapshotDirs, false));

loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
Paths.get(baseDir.toString(), DATA_DIR_2),
Expand Down Expand Up @@ -227,7 +228,8 @@ public void testSnapshotsWithManifests() throws IOException
for (String dataDir : DATA_DIRS)
{
tag1Files.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE1_NAME, TABLE1_ID), Directories.SNAPSHOT_SUBDIR, TAG1));
tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1, tableDirName(TABLE2_NAME, TABLE2_ID), Directories.SNAPSHOT_SUBDIR, TAG2));
// One table is an old pre 2.1 table folder structure
tag2Files.add(createDir(baseDir, dataDir, KEYSPACE_1, TABLE2_NAME, Directories.SNAPSHOT_SUBDIR, TAG2));
tag3Files.add(createDir(baseDir, dataDir, KEYSPACE_2, tableDirName(TABLE3_NAME, TABLE3_ID), Directories.SNAPSHOT_SUBDIR, TAG3));
}

Expand All @@ -254,7 +256,7 @@ public void testSnapshotsWithManifests() throws IOException
Set<TableSnapshot> snapshots = loader.loadSnapshots();
assertThat(snapshots).hasSize(3);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, null, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_2, TABLE3_NAME, TABLE3_ID, TAG3, tag3Ts, null, tag3Files, false));

// Verify snapshot loading for a specific keyspace
Expand All @@ -265,7 +267,7 @@ public void testSnapshotsWithManifests() throws IOException
snapshots = loader.loadSnapshots(KEYSPACE_1);
assertThat(snapshots).hasSize(2);
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE1_NAME, TABLE1_ID, TAG1, tag1Ts, null, tag1Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, TABLE2_ID, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));
assertThat(snapshots).contains(new TableSnapshot(KEYSPACE_1, TABLE2_NAME, null, TAG2, tag2Ts, tag2Ts.plusSeconds(tag2Ttl.toSeconds()), tag2Files, false));

loader = new SnapshotLoader(Arrays.asList(Paths.get(baseDir.toString(), DATA_DIR_1),
Paths.get(baseDir.toString(), DATA_DIR_2),
Expand Down