Value separation (WiscKey, GlideFS-tuned) + hardening

Cargo.toml

@@ -21,7 +21,7 @@ futures-util = { version = "0.3", features = ["sink"] }
 http = "1.0"
 memchr = "2"
 mimalloc = { version = "0.1", features = ["v2"] }
-monoio = { version = "0.2", features = ["io-uring", "unlinkat", "renameat", "sync"] }
+monoio = { version = "0.2", features = ["io-uring", "unlinkat", "renameat", "mkdirat", "sync"] }
 monoio-codec = "0.3"
 monoio-http = "0.3"
 monoio-rustls = "0.4"

crates/engine/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 rust-version.workspace = true
 
 [dependencies]
+blake3 = "1"
 bytes.workspace = true
 crc-fast.workspace = true
 futures-channel.workspace = true
@@ -20,6 +21,7 @@ tracing.workspace = true
 
 [dev-dependencies]
 divan = "0.1"
+libc = "0.2"
 tempfile = "3"
 
 [[bench]]

crates/engine/src/lib.rs

@@ -3,4 +3,5 @@ pub mod error;
 pub mod log;
 pub mod store;
 pub mod types;
+pub mod value_store;
 pub mod watch;

crates/engine/src/log/config.rs

@@ -5,12 +5,23 @@ pub struct LogConfig {
     /// threshold, call `NamespaceLog::rotate_active()` to seal the active file and
     /// open a fresh one. Rotation is operator-controlled and NOT automatic.
     pub rotate_threshold: u64,
+    /// Size-tiered compaction fanout: a level is merged into the next once it
+    /// holds this many runs. Higher = less write-amp, more space-amp. Default 8
+    /// (the measured knee).
+    pub fanout: usize,
+    /// Value-separation threshold in bytes. Values >= this are stored in the
+    /// content-addressed blob store instead of inline in the log, so compaction
+    /// never re-uploads them. Default 128 KiB = one GlideFS block: below a block,
+    /// a blob-per-value wastes space; at/above it, separation collapses write-amp.
+    pub value_sep_threshold: usize,
 }
 
 impl Default for LogConfig {
     fn default() -> Self {
         Self {
             rotate_threshold: 1 << 30, // 1 GiB
+            fanout: 8,
+            value_sep_threshold: 128 * 1024, // 128 KiB = one GlideFS block
         }
     }
 }