Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
581 changes: 382 additions & 199 deletions Cargo.lock

Large diffs are not rendered by default.

25 changes: 12 additions & 13 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,50 +22,49 @@ license = "MIT/Apache-2.0"
borsh = { version = "1.5.7", default-features = false, features = ["derive"] }
bitvec = { version = "1", default-features = false, features = ["alloc"] }
hex = { version = "0.4.3", default-features = false, features = ["alloc"] }
ruint = { version = "1.12.1", default-features = false }
ruint = { version = "1.18.0", default-features = false }
arrayvec = { version = "0.7", default-features = false }
blake3 = { version = "1.5.1", default-features = false }
sha2 = { version = "0.10.6", default-features = false }
anyhow = { version = "1.0.81", features = ["backtrace"] }
parking_lot = { version = "0.12.3", features = ["arc_lock", "send_guard"] }
threadpool = "1.8.1"
twox-hash = "2.1.0"
twox-hash = { version = "2.1.0", default-features = false, features = ["std", "xxhash3_64"] }
fxhash = "0.2.1"
dashmap = "5.5.3"
crossbeam = "0.8.4"
crossbeam-channel = "0.5.13"
crossbeam-channel = "0.5.15"
slab = "0.4.9"
rand = "0.8.5"
rand = "0.10.1"
ahash = "0.8.11"
imbl = "3.0.0"
lru = "0.12.3"
lru = "0.18.0"
libc = "0.2.155"
criterion = { version = "0.3" }
thread_local = "1.1.8"
cfg-if = "1.0.0"
io-uring = "0.6.4"
loom = { version = "0.7", features = ["checkpoint"] }
rand_pcg = "0.3.1"
rand_pcg = "0.10.2"
hex-literal = "0.4"
tempfile = "3.8.1"
lazy_static = "1.5.0"
quickcheck = "1.0.3"
quickcheck = "1.1.0"
nix = { version = "0.29", features = ["process"] }
serde = { version = "1.0.216", default-features = false, features = ["derive"] }
bincode = "1.3.3"
tokio = { version = "1.42.0", features = ["full"] }
tokio-util = { version = "0.7.13", features = ["codec"] }
tokio-stream = "0.1.17"
tokio = { version = "1.52.2", features = ["full"] }
tokio-util = { version = "0.7.18", features = ["codec"] }
tokio-stream = "0.1.18"
futures = "0.3.31"
tokio-serde = { version = "0.9.0", features = ["bincode"] }
tracing = { version = "0.1.41", features = ["attributes"] }
tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
futures-util = "0.3.31"
clap = { version = "4.5.23", features = ["derive"] }
which = "4"
fuser = { version = "0.15.1", features = ["abi-7-23"] }
log = "0.4.22"
rand_distr = "0.4.3"
rand_distr = "0.6.0"
env_logger = "0.11.6"
digest = { version = "0.10.7" }

Expand Down
6 changes: 3 additions & 3 deletions nomt/src/beatree/benches.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::beatree::{
branch::node::benches::*, leaf::node::benches::*, ops::benches::*, ops::bit_ops::benches::*,
Key,
};
use rand::RngCore;
use rand::Rng as _;

pub fn beatree_benchmark(c: &mut criterion::Criterion) {
separate_benchmark(c);
Expand All @@ -19,7 +19,7 @@ pub fn beatree_benchmark(c: &mut criterion::Criterion) {

// returns two keys a and b where b > a and b shares the first n bits with a
pub fn get_key_pair(shared_bytes: usize) -> (Key, Key) {
let mut rand = rand::thread_rng();
let mut rand = rand::rng();
let mut a = [0; 32];
rand.fill_bytes(&mut a[0..shared_bytes]);

Expand All @@ -32,7 +32,7 @@ pub fn get_key_pair(shared_bytes: usize) -> (Key, Key) {

// Get a vector containing `n` random keys that share the first `shared_bytes`
pub fn get_keys(shared_bytes: usize, n: usize) -> Vec<Key> {
let mut rand = rand::thread_rng();
let mut rand = rand::rng();
let mut prefix = [0; 32];
rand.fill_bytes(&mut prefix[0..shared_bytes]);

Expand Down
6 changes: 3 additions & 3 deletions nomt/src/beatree/leaf/node.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,11 +258,11 @@ pub mod benches {
io::PagePool,
};
use criterion::{BatchSize, BenchmarkId, Criterion};
use rand::Rng;
use rand::RngExt;

pub fn leaf_search_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("search_leaf");
let mut rand = rand::thread_rng();
let mut rand = rand::rng();

let page_pool = PagePool::new();

Expand All @@ -284,7 +284,7 @@ pub mod benches {
group.bench_function(BenchmarkId::new("full_leaf", format!("{}-keys", n)), |b| {
b.iter_batched(
|| {
let index = rand.gen_range(0..keys.len());
let index = rand.random_range(0..keys.len());
keys[index].clone()
},
|key| leaf.get(&key),
Expand Down
4 changes: 2 additions & 2 deletions nomt/src/beatree/ops/bit_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ pub fn bitwise_memcpy(
pub mod benches {
use crate::beatree::benches::get_key_pair;
use criterion::{BenchmarkId, Criterion};
use rand::RngCore;
use rand::Rng as _;

pub fn separate_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("separate");
Expand All @@ -308,7 +308,7 @@ pub mod benches {
};

for prefix_bits in [0, 1, 4, 7, 8, 9, 12, 15, 18] {
let mut rand = rand::thread_rng();
let mut rand = rand::rng();
// 50 to extract multiples of 8 bytes for the separator
let mut key = [0; 50];
rand.fill_bytes(&mut key);
Expand Down
6 changes: 3 additions & 3 deletions nomt/src/beatree/ops/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,11 +158,11 @@ pub mod benches {
io::{PagePool, PAGE_SIZE},
};
use criterion::{BenchmarkId, Criterion};
use rand::Rng;
use rand::RngExt;

pub fn search_branch_benchmark(c: &mut Criterion) {
let mut group = c.benchmark_group("search_branch");
let mut rand = rand::thread_rng();
let mut rand = rand::rng();
let page_pool = PagePool::new();

for prefix_len_bytes in [1, 4, 8, 12, 16] {
Expand Down Expand Up @@ -196,7 +196,7 @@ pub mod benches {
|b| {
b.iter_batched(
|| {
let index = rand.gen_range(0..separators.len());
let index = rand.random_range(0..separators.len());
separators[index].1.clone()
},
|separator| super::search_branch(&branch, separator),
Expand Down
10 changes: 5 additions & 5 deletions nomt/src/beatree/ops/update/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use crate::{
};
use lazy_static::lazy_static;
use quickcheck::{Arbitrary, Gen, QuickCheck, TestResult};
use rand::{Rng, SeedableRng};
use rand::{RngExt, SeedableRng};
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::fs::File;
Expand Down Expand Up @@ -119,7 +119,7 @@ fn init_beatree() -> TreeData {
let initial_items: BTreeMap<[u8; 32], Vec<u8>> = KEYS
.iter()
.cloned()
.map(|key| (key, vec![170u8; rng.gen_range(500..MAX_LEAF_VALUE_SIZE)]))
.map(|key| (key, vec![170u8; rng.random_range(500..MAX_LEAF_VALUE_SIZE)]))
.collect();

let leaf_store = Store::open(&PAGE_POOL, ln_fd.clone(), PageNumber(1), None).unwrap();
Expand Down Expand Up @@ -392,7 +392,7 @@ fn leaf_stage_inner(input: StageInputs) -> TestResult {
fn leaf_stage() {
let test_result = std::panic::catch_unwind(|| {
QuickCheck::new()
.gen(Gen::new(LEAF_STAGE_CHANGESET_AVG_SIZE))
.rng(Gen::new(LEAF_STAGE_CHANGESET_AVG_SIZE))
.max_tests(MAX_TESTS)
.quickcheck(leaf_stage_inner as fn(_) -> TestResult)
});
Expand All @@ -414,7 +414,7 @@ fn init_bbn_index(separators: &[[u8; 32]]) -> Index {
let mut bbn_pn = 0;

while used_separators < separators.len() {
let body_size_target = rng.gen_range(BRANCH_MERGE_THRESHOLD..BRANCH_NODE_BODY_SIZE);
let body_size_target = rng.random_range(BRANCH_MERGE_THRESHOLD..BRANCH_NODE_BODY_SIZE);

let branch_node = make_branch_until(
&mut separators[used_separators..].iter().cloned(),
Expand Down Expand Up @@ -588,7 +588,7 @@ fn branch_stage_inner(input: StageInputs) -> TestResult {
fn branch_stage() {
let test_result = std::panic::catch_unwind(|| {
QuickCheck::new()
.gen(Gen::new(BRANCH_STAGE_CHANGESET_AVG_SIZE))
.rng(Gen::new(BRANCH_STAGE_CHANGESET_AVG_SIZE))
.max_tests(MAX_TESTS)
.quickcheck(branch_stage_inner as fn(_) -> TestResult)
});
Expand Down
4 changes: 1 addition & 3 deletions nomt/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,7 @@ pub struct Options {
impl Options {
/// Create a new `Options` instance with the default values and a random bitbox seed.
pub fn new() -> Self {
use rand::Rng as _;
let mut bitbox_seed = [0u8; 16];
rand::rngs::OsRng.fill(&mut bitbox_seed);
let bitbox_seed: [u8; 16] = rand::random();

Self {
path: PathBuf::from("nomt_db"),
Expand Down
2 changes: 1 addition & 1 deletion nomt/tests/common/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use std::{
pub fn account_path(id: u64) -> KeyPath {
// KeyPaths must be uniformly distributed, but we don't want to spend time on a good hash. So
// the next best option is to use a PRNG seeded with the id.
use rand::{RngCore as _, SeedableRng as _};
use rand::{Rng as _, SeedableRng as _};
let mut seed = [0; 16];
seed[0..8].copy_from_slice(&id.to_le_bytes());
let mut rng = rand_pcg::Lcg64Xsh32::from_seed(seed);
Expand Down
2 changes: 1 addition & 1 deletion nomt/tests/fill_and_empty.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
mod common;
use common::Test;
use rand::{prelude::SliceRandom, Rng, SeedableRng};
use rand::{prelude::SliceRandom, Rng, RngExt, SeedableRng};
use std::time::{SystemTime, UNIX_EPOCH};

fn seed() -> [u8; 16] {
Expand Down
1 change: 0 additions & 1 deletion torture/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ libc.workspace = true
anyhow.workspace = true
cfg-if.workspace = true
serde.workspace = true
bincode.workspace = true
nomt = { path = "../nomt" }
tokio.workspace = true
tokio-util.workspace = true
Expand Down
48 changes: 24 additions & 24 deletions torture/src/supervisor/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use super::{
swarm::{self, SwarmFeatures},
ResourceExhaustion,
};
use rand::{Rng, RngCore};
use rand::prelude::*;
use std::sync::{Arc, Mutex};

/// Percentage of the assigned space to the workload that will be
Expand Down Expand Up @@ -165,14 +165,14 @@ impl WorkloadConfiguration {
preallocate_ht: false,
prepopulate_page_cache: false,
bitbox_seed,
avg_commit_size: rng.gen_range(1..=(MAX_COMMIT_SIZE / 2)),
avg_value_len: rng.gen_range(1..=(MAX_VALUE_LEN / 2)),
avg_overflow_value_len: rng.gen_range(MAX_VALUE_LEN..=(MAX_OVERFLOW_VALUE_LEN / 2)),
commit_concurrency: rng.gen_range(1..=MAX_COMMIT_CONCURRENCY),
io_workers: rng.gen_range(1..=MAX_IO_WORKERS),
page_cache_size: rng.gen_range(1..=MAX_IN_MEMORY_CACHE_SIZE),
leaf_cache_size: rng.gen_range(1..=MAX_IN_MEMORY_CACHE_SIZE),
page_cache_upper_levels: rng.gen_range(0..=MAX_PAGE_CACHE_UPPER_LEVELS),
avg_commit_size: rng.random_range(1..=(MAX_COMMIT_SIZE / 2)),
avg_value_len: rng.random_range(1..=(MAX_VALUE_LEN / 2)),
avg_overflow_value_len: rng.random_range(MAX_VALUE_LEN..=(MAX_OVERFLOW_VALUE_LEN / 2)),
commit_concurrency: rng.random_range(1..=MAX_COMMIT_CONCURRENCY),
io_workers: rng.random_range(1..=MAX_IO_WORKERS),
page_cache_size: rng.random_range(1..=MAX_IN_MEMORY_CACHE_SIZE),
leaf_cache_size: rng.random_range(1..=MAX_IN_MEMORY_CACHE_SIZE),
page_cache_upper_levels: rng.random_range(0..=MAX_PAGE_CACHE_UPPER_LEVELS),
// To avoid reaching Bucket Exhaustion, we limit the number of iterations
// with the worst case scenario of every iteration adding `avg_commit_size` new keys.
hashtable_buckets: 0,
Expand Down Expand Up @@ -296,35 +296,35 @@ impl WorkloadConfiguration {
pub fn apply_swarm_feature(&mut self, rng: &mut rand_pcg::Pcg64, feature: SwarmFeatures) {
match feature {
SwarmFeatures::TrickfsENOSPC => {
self.enospc_on = rng.gen_range(0.01..=1.00);
self.enospc_off = rng.gen_range(0.01..=1.00);
self.enospc_on = rng.random_range(0.01..=1.00);
self.enospc_off = rng.random_range(0.01..=1.00);
}
SwarmFeatures::TrickfsLatencyInjection => {
self.latency_on = rng.gen_range(0.01..=1.00);
self.latency_off = rng.gen_range(0.01..=1.00);
self.latency_on = rng.random_range(0.01..=1.00);
self.latency_off = rng.random_range(0.01..=1.00);
}
SwarmFeatures::EnsureChangeset => self.ensure_changeset = true,
SwarmFeatures::SampleSnapshot => self.sample_snapshot = true,
SwarmFeatures::WarmUp => self.warm_up = true,
SwarmFeatures::PreallocateHt => self.preallocate_ht = true,
SwarmFeatures::Read => {
self.reads = rng.gen_range(0.01..1.00);
self.read_concurrency = rng.gen_range(1..=64);
self.read_existing_key = rng.gen_range(0.01..1.00);
self.reads = rng.random_range(0.01..1.00);
self.read_concurrency = rng.random_range(1..=64);
self.read_existing_key = rng.random_range(0.01..1.00);
}
SwarmFeatures::Rollback => {
self.rollback = rng.gen_range(0.01..1.00);
self.rollback = rng.random_range(0.01..1.00);
// During scheduling of rollbacks the lower bound is 1
// thus max must be at least 2 to avoid creating an empty range.
self.max_rollback_commits = rng.gen_range(2..100);
self.max_rollback_commits = rng.random_range(2..100);
}
SwarmFeatures::RollbackCrash => self.rollback_crash = rng.gen_range(0.01..1.00),
SwarmFeatures::CommitCrash => self.commit_crash = rng.gen_range(0.01..1.00),
SwarmFeatures::RollbackCrash => self.rollback_crash = rng.random_range(0.01..1.00),
SwarmFeatures::CommitCrash => self.commit_crash = rng.random_range(0.01..1.00),
SwarmFeatures::PrepopulatePageCache => self.prepopulate_page_cache = true,
SwarmFeatures::NewKeys => self.new_key = rng.gen_range(0.01..=1.00),
SwarmFeatures::DeleteKeys => self.delete_key = rng.gen_range(0.01..=1.00),
SwarmFeatures::UpdateKeys => self.update_key = rng.gen_range(0.01..=1.00),
SwarmFeatures::OverflowValues => self.overflow = rng.gen_range(0.01..=1.00),
SwarmFeatures::NewKeys => self.new_key = rng.random_range(0.01..=1.00),
SwarmFeatures::DeleteKeys => self.delete_key = rng.random_range(0.01..=1.00),
SwarmFeatures::UpdateKeys => self.update_key = rng.random_range(0.01..=1.00),
SwarmFeatures::OverflowValues => self.overflow = rng.random_range(0.01..=1.00),
}
}
}
6 changes: 3 additions & 3 deletions torture/src/supervisor/resource.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::{
path::{Path, PathBuf},
};

use rand::{Rng, SeedableRng};
use rand::{prelude::*, SeedableRng};

/// 1GiB is the minimum amount of space that can be assigned to a workload.
const MIN_ASSIGNED_SPACE: u64 = 1 * (1 << 30);
Expand Down Expand Up @@ -82,7 +82,7 @@ impl ResourceAllocator {

// Force to allocate bigger chunk of memory initially.
let min = std::cmp::min(MIN_ASSIGNED_SPACE, avail_disk / 2);
let assigned_disk = self.rng.gen_range(min..avail_disk);
let assigned_disk = self.rng.random_range(min..avail_disk);

// Assign memory
let mut avail_memory = self.max_memory_avail - self.total_assigned_memory;
Expand Down Expand Up @@ -114,7 +114,7 @@ impl ResourceAllocator {
}

avail_memory = std::cmp::min(avail_memory, MAX_ASSIGNED_ONLY_MEMORY);
let assigned_memory = self.rng.gen_range(MIN_ASSIGNED_SPACE..avail_memory);
let assigned_memory = self.rng.random_range(MIN_ASSIGNED_SPACE..avail_memory);

self.total_assigned_memory += assigned_memory;
self.assigned.push((
Expand Down
8 changes: 4 additions & 4 deletions torture/src/supervisor/swarm.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use rand::Rng;
use rand::RngExt;

pub enum SwarmFeatures {
/// Trigger on and off trickfs to return ENOSPC.
Expand Down Expand Up @@ -59,7 +59,7 @@ pub fn new_features_set(rng: &mut rand_pcg::Pcg64) -> Vec<SwarmFeatures> {

// Features removal mechanism -> coin tossing for almost every feature.
for idx in (0..features.len()).rev() {
if rng.gen_bool(0.5) {
if rng.random_bool(0.5) {
features.remove(idx);
}
}
Expand All @@ -71,10 +71,10 @@ pub fn new_features_set(rng: &mut rand_pcg::Pcg64) -> Vec<SwarmFeatures> {
//
// The probability of using Trickfs is 10% (= p*p + 2 * (p * (1-p))).
let p = 0.052;
if rng.gen_bool(p) {
if rng.random_bool(p) {
features.push(SwarmFeatures::TrickfsLatencyInjection);
}
if rng.gen_bool(p) {
if rng.random_bool(p) {
features.push(SwarmFeatures::TrickfsENOSPC);
}

Expand Down
Loading
Loading